diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index a96d54d5b6..748999f872 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -54,6 +54,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.calcite.rel.RelNode; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -166,8 +167,10 @@ import org.apache.hadoop.hive.ql.metadata.formatting.TextMetaDataTable; import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.CalcitePlanner; import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; +import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.PreInsertTableDesc; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -1259,6 +1262,32 @@ private int alterMaterializedView(Hive db, AlterMaterializedViewDesc alterMVDesc // This is a noop, return successfully return 0; } + if (alterMVDesc.isRewriteEnable()) { + try { + final QueryState qs = + new QueryState.Builder().withHiveConf(conf).build(); + final CalcitePlanner planner = new CalcitePlanner(qs); + final Context ctx = new Context(conf); + ctx.setIsLoadingMaterializedView(true); + planner.initCtx(ctx); + planner.init(false); + final RelNode plan = planner.genLogicalPlan(ParseUtils.parse(mv.getViewExpandedText())); + if (plan == null) { + String msg = "Cannot enable automatic rewriting for materialized view. " + + "Materialized view definition could not be parsed by CBO."; + if (ctx.getCboInfo() != null) { + msg += " " + ctx.getCboInfo(); + } + throw new HiveException(msg); + } + if (!planner.isValidAutomaticRewritingMaterialization()) { + throw new HiveException("Cannot enable rewriting for materialized view. " + + planner.getInvalidAutomaticRewritingMaterializationReason()); + } + } catch (Exception e) { + throw new HiveException(e); + } + } mv.setRewriteEnabled(alterMVDesc.isRewriteEnable()); break; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java similarity index 81% rename from ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java rename to ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java index df216e7555..ed99163069 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOpMaterializationValidator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptMaterializationValidator.java @@ -18,10 +18,8 @@ package org.apache.hadoop.hive.ql.optimizer.calcite; -import java.util.List; - import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -39,11 +37,8 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.Util; - import org.apache.hadoop.hive.metastore.TableType; - import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter; @@ -54,7 +49,6 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,12 +58,13 @@ * - References to temporary or external tables * - References to non-determinisitc functions. */ -public class HiveRelOpMaterializationValidator extends HiveRelShuttleImpl { - static final Logger LOG = LoggerFactory.getLogger(HiveRelOpMaterializationValidator.class); +public class HiveRelOptMaterializationValidator extends HiveRelShuttleImpl { + static final Logger LOG = LoggerFactory.getLogger(HiveRelOptMaterializationValidator.class); - protected String invalidMaterializationReason; + protected String queryCachingInvalidReason; + protected String automaticRewritingInvalidReason; - public void validateQueryMaterialization(RelNode relNode) { + public void validate(RelNode relNode) { try { relNode.accept(this); } catch (Util.FoundOne e) { @@ -86,7 +81,6 @@ public RelNode visit(TableScan scan) { if (tab.isTemporary()) { fail(tab.getTableName() + " is a temporary table"); } - TableType tt = tab.getTableType(); if (tab.getTableType() == TableType.EXTERNAL_TABLE) { fail(tab.getFullyQualifiedName() + " is an external table"); } @@ -114,6 +108,9 @@ public RelNode visit(HiveFilter filter) { @Override public RelNode visit(HiveJoin join) { + if (join.getJoinType() != JoinRelType.INNER) { + setAutomaticRewritingInvalidReason(join.getJoinType() + " join type is not supported by rewriting algorithm."); + } checkExpr(join.getCondition()); return super.visit(join); } @@ -126,6 +123,7 @@ public RelNode visit(HiveAggregate aggregate) { @Override public RelNode visit(RelNode node) { + setAutomaticRewritingInvalidReason(node); // There are several Hive RelNode types which do not have their own visit() method // defined in the HiveRelShuttle interface, which need to be handled appropriately here. // Per jcamachorodriguez we should not encounter HiveMultiJoin/HiveSortExchange @@ -148,6 +146,7 @@ public RelNode visit(RelNode node) { @Override public RelNode visit(TableFunctionScan scan) { + setAutomaticRewritingInvalidReason(scan); checkExpr(scan.getCall()); return super.visit(scan); } @@ -254,13 +253,15 @@ private RelNode visit(HiveIntersect intersect) { } private void fail(String reason) { - setInvalidMaterializationReason(reason); + setQueryCachingInvalidReason(reason); + setAutomaticRewritingInvalidReason(reason); throw Util.FoundOne.NULL; } private RelNode fail(RelNode node) { - setInvalidMaterializationReason("Unsupported RelNode type " + node.getRelTypeName() + + setQueryCachingInvalidReason("Unsupported RelNode type " + node.getRelTypeName() + " encountered in the query plan"); + setAutomaticRewritingInvalidReason(node); throw Util.FoundOne.NULL; } @@ -271,15 +272,36 @@ private void checkExpr(RexNode expr) { } } - public String getInvalidMaterializationReason() { - return invalidMaterializationReason; + public String getQueryCachingInvalidReason() { + return queryCachingInvalidReason; + } + + public void setQueryCachingInvalidReason(String queryCachingInvalidReason) { + this.queryCachingInvalidReason = queryCachingInvalidReason; } - public void setInvalidMaterializationReason(String invalidMaterializationReason) { - this.invalidMaterializationReason = invalidMaterializationReason; + public boolean isValidForQueryCaching() { + return queryCachingInvalidReason == null; + } + + public String getAutomaticRewritingInvalidReason() { + return automaticRewritingInvalidReason; + } + + public void setAutomaticRewritingInvalidReason(String automaticRewritingInvalidReason) { + if (isValidForAutomaticRewriting()) { + this.automaticRewritingInvalidReason = automaticRewritingInvalidReason; + } + } + + public void setAutomaticRewritingInvalidReason(RelNode node) { + if (isValidForAutomaticRewriting()) { + this.automaticRewritingInvalidReason = "Unsupported RelNode type " + node.getRelTypeName() + + " encountered in the query plan"; + } } - public boolean isValidMaterialization() { - return invalidMaterializationReason == null; + public boolean isValidForAutomaticRewriting() { + return automaticRewritingInvalidReason == null; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index aef1d4ab2a..f54c0416c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -149,7 +149,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider; import org.apache.hadoop.hive.ql.optimizer.calcite.HivePlannerContext; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; -import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOpMaterializationValidator; +import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptMaterializationValidator; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl; import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable; @@ -384,8 +384,10 @@ public RelNode genLogicalPlan(ASTNode ast) throws SemanticException { if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) { queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } - runCBO = canCBOHandleAst(queryForCbo, getQB(), cboCtx); + Pair pairCanCBOHandleReason = canCBOHandleAst(queryForCbo, getQB(), cboCtx); + runCBO = pairCanCBOHandleReason.left; if (!runCBO) { + ctx.setCboInfo("Plan not optimized by CBO because the statement " + pairCanCBOHandleReason.right); return null; } profilesCBO = obtainCBOProfiles(queryProperties); @@ -448,7 +450,8 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept if (cboCtx.type == PreCboCtx.Type.CTAS || cboCtx.type == PreCboCtx.Type.VIEW) { queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query } - runCBO = canCBOHandleAst(queryForCbo, getQB(), cboCtx); + Pair pairCanCBOHandleReason = canCBOHandleAst(queryForCbo, getQB(), cboCtx); + runCBO = pairCanCBOHandleReason.left; if (queryProperties.hasMultiDestQuery()) { handleMultiDestQuery(ast, cboCtx); } @@ -609,7 +612,13 @@ else if (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || isMissingStats } } } else { - this.ctx.setCboInfo("Plan not optimized by CBO."); + String msg; + if (pairCanCBOHandleReason.right != null) { + msg = "Plan not optimized by CBO because the statement " + pairCanCBOHandleReason.right; + } else { + msg = "Plan not optimized by CBO."; + } + this.ctx.setCboInfo(msg); skipCalcitePlan = true; } } @@ -812,7 +821,7 @@ public Object post(Object t) { * If top level QB is query then everything below it must also be * Query. */ - boolean canCBOHandleAst(ASTNode ast, QB qb, PreCboCtx cboCtx) { + Pair canCBOHandleAst(ASTNode ast, QB qb, PreCboCtx cboCtx) { int root = ast.getToken().getType(); boolean needToLogMessage = STATIC_LOG.isInfoEnabled(); boolean isSupportedRoot = root == HiveParser.TOK_QUERY || root == HiveParser.TOK_EXPLAIN @@ -824,39 +833,39 @@ boolean canCBOHandleAst(ASTNode ast, QB qb, PreCboCtx cboCtx) { boolean noBadTokens = HiveCalciteUtil.validateASTForUnsupportedTokens(ast); boolean result = isSupportedRoot && isSupportedType && noBadTokens; + String msg = ""; if (!result) { + if (!isSupportedRoot) { + msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; "; + } + if (!isSupportedType) { + msg += "is not a query with at least one source table " + + " or there is a subquery without a source table, or CTAS, or insert; "; + } + if (!noBadTokens) { + msg += "has unsupported tokens; "; + } + if (msg.isEmpty()) { + msg += "has some unspecified limitations; "; + } + msg = msg.substring(0, msg.length() - 2); if (needToLogMessage) { - String msg = ""; - if (!isSupportedRoot) { - msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; "; - } - if (!isSupportedType) { - msg += "is not a query with at least one source table " - + " or there is a subquery without a source table, or CTAS, or insert; "; - } - if (!noBadTokens) { - msg += "has unsupported tokens; "; - } - - if (msg.isEmpty()) { - msg += "has some unspecified limitations; "; - } - STATIC_LOG.info("Not invoking CBO because the statement " - + msg.substring(0, msg.length() - 2)); + STATIC_LOG.info("Not invoking CBO because the statement " + msg); } - return false; + return Pair.of(false, msg); } + // Now check QB in more detail. canHandleQbForCbo returns null if query can // be handled. - String msg = CalcitePlanner.canHandleQbForCbo(queryProperties, conf, true, needToLogMessage, qb); + msg = CalcitePlanner.canHandleQbForCbo(queryProperties, conf, true, needToLogMessage, qb); if (msg == null) { - return true; + return Pair.of(true, msg); } + msg = msg.substring(0, msg.length() - 2); if (needToLogMessage) { - STATIC_LOG.info("Not invoking CBO because the statement " - + msg.substring(0, msg.length() - 2)); + STATIC_LOG.info("Not invoking CBO because the statement " + msg); } - return false; + return Pair.of(false, msg); } /** @@ -1741,16 +1750,6 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu } perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Plan generation"); - // Validate query materialization (materialized views, query results caching. - // This check needs to occur before constant folding, which may remove some - // function calls from the query plan. - HiveRelOpMaterializationValidator matValidator = new HiveRelOpMaterializationValidator(); - matValidator.validateQueryMaterialization(calciteGenPlan); - if (!matValidator.isValidMaterialization()) { - String reason = matValidator.getInvalidMaterializationReason(); - setInvalidQueryMaterializationReason(reason); - } - // Create executor RexExecutor executorProvider = new HiveRexExecutorImpl(optCluster); calciteGenPlan.getCluster().getPlanner().setExecutor(executorProvider); @@ -1776,6 +1775,20 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu calciteGenPlan = HiveRelDecorrelator.decorrelateQuery(calciteGenPlan); LOG.debug("Plan after decorrelation:\n" + RelOptUtil.toString(calciteGenPlan)); + // Validate query materialization for query results caching. This check needs + // to occur before constant folding, which may remove some function calls + // from the query plan. + // In addition, if it is a materialized view creation and we are enabling it + // for rewriting, it should pass all checks done for query results caching + // and on top of that we should check that it only contains operators that + // are supported by the rewriting algorithm. + HiveRelOptMaterializationValidator materializationValidator = new HiveRelOptMaterializationValidator(); + materializationValidator.validate(calciteGenPlan); + setInvalidQueryCachingReason( + materializationValidator.getQueryCachingInvalidReason()); + setInvalidAutomaticRewritingMaterializationReason( + materializationValidator.getAutomaticRewritingInvalidReason()); + // 2. Apply pre-join order optimizations calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan, mdProvider.getMetadataProvider(), executorProvider); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b330d710a1..433a83e97c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -396,7 +396,8 @@ HiveParser.TOK_ORDERBY, HiveParser.TOK_WINDOWSPEC, HiveParser.TOK_CLUSTERBY, HiveParser.TOK_DISTRIBUTEBY, HiveParser.TOK_SORTBY); - private String invalidQueryMaterializationReason; + private String invalidQueryCachingReason; + private String invalidAutomaticRewritingMaterializationReason; private static final CommonToken SELECTDI_TOKEN = new ImmutableCommonToken(HiveParser.TOK_SELECTDI, "TOK_SELECTDI"); @@ -13716,6 +13717,21 @@ private void validateCreateView() } } + if (createVwDesc.isMaterialized() && createVwDesc.isRewriteEnabled()) { + if (!ctx.isCboSucceeded()) { + String msg = "Cannot enable automatic rewriting for materialized view. " + + "Materialized view definition could not be parsed by CBO."; + if (ctx.getCboInfo() != null) { + msg += " " + ctx.getCboInfo(); + } + throw new SemanticException(msg); + } + if (!isValidAutomaticRewritingMaterialization()) { + throw new SemanticException("Cannot enable rewriting for materialized view. " + + getInvalidAutomaticRewritingMaterializationReason()); + } + } + // ALTER VIEW AS SELECT requires the view must exist if (createVwDesc.getIsAlterViewAs() && oldView == null) { String viewNotExistErrorMsg = @@ -15033,8 +15049,8 @@ private boolean queryCanBeCached() { return false; } - if (!isValidQueryMaterialization()) { - LOG.info("Not eligible for results caching - {}", getInvalidQueryMaterializationReason()); + if (!isValidQueryCaching()) { + LOG.info("Not eligible for results caching - {}", getInvalidQueryCachingReason()); QueryResultsCache.incrementMetric(MetricsConstant.QC_INVALID_FOR_CACHING); return false; } @@ -15130,17 +15146,31 @@ public ColsAndTypes(String cols, String colTypes) { public String colTypes; } - public String getInvalidQueryMaterializationReason() { - return invalidQueryMaterializationReason; + public String getInvalidAutomaticRewritingMaterializationReason() { + return invalidAutomaticRewritingMaterializationReason; + } + + public void setInvalidAutomaticRewritingMaterializationReason( + String invalidAutomaticRewritingMaterializationReason) { + this.invalidAutomaticRewritingMaterializationReason = + invalidAutomaticRewritingMaterializationReason; + } + + public boolean isValidAutomaticRewritingMaterialization() { + return (invalidAutomaticRewritingMaterializationReason == null); + } + + public String getInvalidQueryCachingReason() { + return invalidQueryCachingReason; } - public void setInvalidQueryMaterializationReason( + public void setInvalidQueryCachingReason( String invalidQueryMaterializationReason) { - this.invalidQueryMaterializationReason = invalidQueryMaterializationReason; + this.invalidQueryCachingReason = invalidQueryMaterializationReason; } - public boolean isValidQueryMaterialization() { - return (invalidQueryMaterializationReason == null); + public boolean isValidQueryCaching() { + return (invalidQueryCachingReason == null); } protected enum MaterializationRebuildMode { diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite.q b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite.q new file mode 100644 index 0000000000..d28ff0b6fd --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite.q @@ -0,0 +1,11 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view as select a, b, c from cmv_basetable sort by a; diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite_2.q b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite_2.q new file mode 100644 index 0000000000..ea360ebd06 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_cbo_rewrite_2.q @@ -0,0 +1,13 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view disable rewrite as select a, b, c from cmv_basetable sort by a; + +alter materialized view cmv_mat_view enable rewrite; diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite.q b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite.q new file mode 100644 index 0000000000..204c264812 --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite.q @@ -0,0 +1,11 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b); diff --git a/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite_2.q b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite_2.q new file mode 100644 index 0000000000..7050f1ac9a --- /dev/null +++ b/ql/src/test/queries/clientnegative/materialized_view_no_supported_op_rewrite_2.q @@ -0,0 +1,13 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view cmv_mat_view disable rewrite as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b); + +alter materialized view cmv_mat_view enable rewrite; diff --git a/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite.q.out b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite.q.out new file mode 100644 index 0000000000..86cb4b5520 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite.q.out @@ -0,0 +1,22 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +FAILED: SemanticException Cannot enable automatic rewriting for materialized view. Materialized view definition could not be parsed by CBO. Plan not optimized by CBO because the statement has sort by diff --git a/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite_2.q.out b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite_2.q.out new file mode 100644 index 0000000000..6d08f02620 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_cbo_rewrite_2.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +PREHOOK: query: create materialized view cmv_mat_view disable rewrite as select a, b, c from cmv_basetable sort by a +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view disable rewrite as select a, b, c from cmv_basetable sort by a +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: alter materialized view cmv_mat_view enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Cannot enable automatic rewriting for materialized view. Materialized view definition could not be parsed by CBO. Plan not optimized by CBO because the statement has sort by diff --git a/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite.q.out b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite.q.out new file mode 100644 index 0000000000..f3341bf585 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite.q.out @@ -0,0 +1,22 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +FAILED: SemanticException Cannot enable rewriting for materialized view. LEFT join type is not supported by rewriting algorithm. diff --git a/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite_2.q.out b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite_2.q.out new file mode 100644 index 0000000000..0b72e3cb83 --- /dev/null +++ b/ql/src/test/results/clientnegative/materialized_view_no_supported_op_rewrite_2.q.out @@ -0,0 +1,37 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.b SCRIPT [] +POSTHOOK: Lineage: cmv_basetable.c SCRIPT [] +Warning: Shuffle Join JOIN[6][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: create materialized view cmv_mat_view disable rewrite as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view disable rewrite as select t1.a from cmv_basetable t1 left outer join cmv_basetable t2 on (t1.a = t1.b) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: alter materialized view cmv_mat_view enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.metadata.HiveException: Cannot enable rewriting for materialized view. LEFT join type is not supported by rewriting algorithm. diff --git a/ql/src/test/results/clientpositive/join0.q.out b/ql/src/test/results/clientpositive/join0.q.out index 2601b1ad86..2b09a435fc 100644 --- a/ql/src/test/results/clientpositive/join0.q.out +++ b/ql/src/test/results/clientpositive/join0.q.out @@ -122,7 +122,7 @@ SELECT src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"cboInfo":"Plan not optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6"}}}}}}}},{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"columns:":["_col0","_col1","_col2","_col3"],"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"cboInfo":"Plan not optimized by CBO because the statement has sort by","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6"}}}}}}}},{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"columns:":["_col0","_col1","_col2","_col3"],"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 6882963599..dfbbeb61e6 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -297,7 +297,7 @@ select src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (XPROD_EDGE), Map 4 (XPROD_EDGE) @@ -3644,7 +3644,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_CTAS1_n1 -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -3703,7 +3703,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_ctas3_n1 -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -3801,7 +3801,7 @@ select src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) @@ -4652,7 +4652,7 @@ from noop(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -4701,7 +4701,7 @@ sort by j.p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Map 1 <- Map 4 (BROADCAST_EDGE) @@ -4768,7 +4768,7 @@ from noop(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -4823,7 +4823,7 @@ from noop(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -4880,7 +4880,7 @@ group by p_mfgr, p_name, p_size POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -4937,7 +4937,7 @@ order by p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) @@ -4987,7 +4987,7 @@ order by p_name, p_size desc) POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5042,7 +5042,7 @@ from noopwithmap(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5097,7 +5097,7 @@ order by p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5152,7 +5152,7 @@ order by p_mfgr DESC, p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5223,7 +5223,7 @@ window w1 as (partition by p_mfgr order by p_name rows between 2 preceding and 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5282,7 +5282,7 @@ order by p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) @@ -5433,7 +5433,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: default@part_4_n1 POSTHOOK: Output: default@part_5_n1 -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5567,7 +5567,7 @@ from noop(on POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5646,7 +5646,7 @@ from noop(on POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) @@ -5727,7 +5727,7 @@ from noop(on POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 1dd9a0ff30..9f5a4b96e8 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -3660,7 +3660,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: cat POSTHOOK: Input: default@src POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement uses scripts Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) diff --git a/ql/src/test/results/clientpositive/parallel_join0.q.out b/ql/src/test/results/clientpositive/parallel_join0.q.out index 2601b1ad86..2b09a435fc 100644 --- a/ql/src/test/results/clientpositive/parallel_join0.q.out +++ b/ql/src/test/results/clientpositive/parallel_join0.q.out @@ -122,7 +122,7 @@ SELECT src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -{"cboInfo":"Plan not optimized by CBO.","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6"}}}}}}}},{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"columns:":["_col0","_col1","_col2","_col3"],"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} +{"cboInfo":"Plan not optimized by CBO because the statement has sort by","STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-2":{"DEPENDENT STAGES":"Stage-1"},"Stage-0":{"DEPENDENT STAGES":"Stage-2"}},"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_0","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_13","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_2","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_6"}}}}}}}},{"TableScan":{"alias:":"src","columns:":["key","value"],"database:":"default","filterExpr:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE","table:":"src","isTempTable:":"false","OperatorId:":"TS_3","children":{"Filter Operator":{"predicate:":"(key < 10) (type: boolean)","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"FIL_14","children":{"Select Operator":{"expressions:":"key (type: string), value (type: string)","columnExprMap:":{"_col0":"key","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_5","children":{"Reduce Output Operator":{"columnExprMap:":{"VALUE._col0":"_col0","VALUE._col1":"_col1"},"sort order:":"","Statistics:":"Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE","value expressions:":"_col0 (type: string), _col1 (type: string)","OperatorId:":"RS_7"}}}}}}}}],"Reduce Operator Tree:":{"Join Operator":{"columnExprMap:":{"_col0":"0:VALUE._col0","_col1":"0:VALUE._col1","_col2":"1:VALUE._col0","_col3":"1:VALUE._col1"},"condition map:":[{"":"Inner Join 0 to 1"}],"keys:":{},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"JOIN_8","children":{"File Output Operator":{"compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe"},"OperatorId:":"FS_15"}}}}}},"Stage-2":{"Map Reduce":{"Map Operator Tree:":[{"TableScan":{"columns:":["_col0","_col1","_col2","_col3"],"OperatorId:":"TS_16","children":{"Reduce Output Operator":{"columnExprMap:":{"KEY.reducesinkkey0":"_col0","KEY.reducesinkkey1":"_col1","KEY.reducesinkkey2":"_col2","KEY.reducesinkkey3":"_col3"},"key expressions:":"_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)","sort order:":"++++","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"RS_10"}}}}],"Reduce Operator Tree:":{"Select Operator":{"expressions:":"KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string)","columnExprMap:":{"_col0":"KEY.reducesinkkey0","_col1":"KEY.reducesinkkey1","_col2":"KEY.reducesinkkey2","_col3":"KEY.reducesinkkey3"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_11","children":{"File Output Operator":{"compressed:":"false","Statistics:":"Num rows: 27556 Data size: 612872 Basic stats: COMPLETE Column stats: NONE","table:":{"input format:":"org.apache.hadoop.mapred.SequenceFileInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"OperatorId:":"FS_12"}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"ListSink":{"OperatorId:":"LIST_SINK_17"}}}}}} Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: SELECT src1.key as k1, src1.value as v1, src2.key as k2, src2.value as v2 FROM diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index 3e395a84ec..0422dc21f4 100644 --- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -263,7 +263,7 @@ select src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in Stage-1 Reducer 2 <- Map 1 (GROUP) @@ -3618,7 +3618,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_CTAS1_n0 -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -3674,7 +3674,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_ctas3_n0 -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -3769,7 +3769,7 @@ select src1.key as k1, src1.value as v1, POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has sort by Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 4 (PARTITION-LEVEL SORT) @@ -4608,7 +4608,7 @@ from noop(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -4657,7 +4657,7 @@ sort by j.p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in Stage-1 Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -4726,7 +4726,7 @@ from noop(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -4781,7 +4781,7 @@ from noop(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -4838,7 +4838,7 @@ group by p_mfgr, p_name, p_size POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -4895,7 +4895,7 @@ order by p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in Stage-1 Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -4949,7 +4949,7 @@ order by p_name, p_size desc) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5004,7 +5004,7 @@ from noopwithmap(on part POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5059,7 +5059,7 @@ order by p_name) POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5114,7 +5114,7 @@ order by p_mfgr DESC, p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5185,7 +5185,7 @@ window w1 as (partition by p_mfgr order by p_name rows between 2 preceding and 2 POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5244,7 +5244,7 @@ order by p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in Stage-1 Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5399,7 +5399,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: default@part_4_n0 POSTHOOK: Output: default@part_5_n0 -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT) @@ -5506,7 +5506,7 @@ from noop(on POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5585,7 +5585,7 @@ from noop(on POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) @@ -5666,7 +5666,7 @@ from noop(on POSTHOOK: type: QUERY POSTHOOK: Input: default@part POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has PTF Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index bac8c09686..30ec550c5b 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -558,7 +558,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: cat POSTHOOK: Input: default@src POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement uses scripts Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index d219478495..de684401f5 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -34,7 +34,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has lateral views Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 25e4672257..69b4ba4c38 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -72,7 +72,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 POSTHOOK: Output: hdfs://### HDFS PATH ### -Plan not optimized by CBO. +Plan not optimized by CBO because the statement has lateral views Stage-0 Fetch Operator