diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 3b1005f85c..02f6bc384b 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -206,6 +206,15 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ load_dyn_part2.q,\ load_dyn_part3.q,\ lvj_mapjoin.q,\ + materialized_view_create_rewrite.q,\ + materialized_view_create_rewrite_3.q,\ + materialized_view_describe.q,\ + materialized_view_rewrite_ssb.q,\ + materialized_view_create.q,\ + materialized_view_create_rewrite_2.q,\ + materialized_view_create_rewrite_multi_db.q,\ + materialized_view_drop.q,\ + materialized_view_rewrite_ssb_2.q\ mapjoin2.q,\ mapjoin3.q,\ mapjoin_decimal.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index e7af5e004f..b3c04c9264 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -5039,7 +5039,7 @@ private int createTableLike(Hive db, CreateTableLikeDesc crtTbl) throws Exceptio */ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { Table oldview = db.getTable(crtView.getViewName(), false); - if (crtView.getOrReplace() && oldview != null) { + if (crtView.isReplace() && oldview != null) { if (!crtView.isMaterialized()) { // replace existing view // remove the existing partition columns from the field schema @@ -5067,87 +5067,12 @@ private int createView(Hive db, CreateViewDesc crtView) throws HiveException { } addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); } else { - // This is a replace, so we need an exclusive lock + // This is a replace/rebuild, so we need an exclusive lock addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_EXCLUSIVE)); } } else { // create new view - Table tbl = db.newTable(crtView.getViewName()); - tbl.setViewOriginalText(crtView.getViewOriginalText()); - tbl.setViewExpandedText(crtView.getViewExpandedText()); - if (crtView.isMaterialized()) { - tbl.setRewriteEnabled(crtView.isRewriteEnabled()); - tbl.setTableType(TableType.MATERIALIZED_VIEW); - } else { - tbl.setTableType(TableType.VIRTUAL_VIEW); - } - tbl.setSerializationLib(null); - tbl.clearSerDeInfo(); - tbl.setFields(crtView.getSchema()); - if (crtView.getComment() != null) { - tbl.setProperty("comment", crtView.getComment()); - } - if (crtView.getTblProps() != null) { - tbl.getTTable().getParameters().putAll(crtView.getTblProps()); - } - - if (crtView.getPartCols() != null) { - tbl.setPartCols(crtView.getPartCols()); - } - - if (crtView.getInputFormat() != null) { - tbl.setInputFormatClass(crtView.getInputFormat()); - } - - if (crtView.getOutputFormat() != null) { - tbl.setOutputFormatClass(crtView.getOutputFormat()); - } - - if (crtView.isMaterialized()) { - if (crtView.getLocation() != null) { - tbl.setDataLocation(new Path(crtView.getLocation())); - } - - if (crtView.getStorageHandler() != null) { - tbl.setProperty( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, - crtView.getStorageHandler()); - } - HiveStorageHandler storageHandler = tbl.getStorageHandler(); - - /* - * If the user didn't specify a SerDe, we use the default. - */ - String serDeClassName; - if (crtView.getSerde() == null) { - if (storageHandler == null) { - serDeClassName = PlanUtils.getDefaultSerDe().getName(); - LOG.info("Default to {} for materialized view {}", serDeClassName, - crtView.getViewName()); - } else { - serDeClassName = storageHandler.getSerDeClass().getName(); - LOG.info("Use StorageHandler-supplied {} for materialized view {}", - serDeClassName, crtView.getViewName()); - } - } else { - // let's validate that the serde exists - serDeClassName = crtView.getSerde(); - DDLTask.validateSerDe(serDeClassName, conf); - } - tbl.setSerializationLib(serDeClassName); - - // To remain consistent, we need to set input and output formats both - // at the table level and the storage handler level. - tbl.setInputFormatClass(crtView.getInputFormat()); - tbl.setOutputFormatClass(crtView.getOutputFormat()); - if (crtView.getInputFormat() != null && !crtView.getInputFormat().isEmpty()) { - tbl.getSd().setInputFormat(tbl.getInputFormatClass().getName()); - } - if (crtView.getOutputFormat() != null && !crtView.getOutputFormat().isEmpty()) { - tbl.getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); - } - } - + Table tbl = crtView.toTable(conf); db.createTable(tbl, crtView.getIfNotExists()); // Add to cache if it is a materialized view if (tbl.isMaterializedView()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 67739a1db9..be1d4b8cee 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1500,14 +1500,33 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, statsWork = new BasicStatsWork(mvWork.getLoadFileWork()); truncate = true; - if (mvWork.getLoadFileWork().getCtasCreateTableDesc() == null) { - throw new RuntimeException("unexpected; this should be a CTAS - however no desc present"); - } - try { - table = mvWork.getLoadFileWork().getCtasCreateTableDesc().toTable(hconf); - } catch (HiveException e) { - LOG.debug("can't pre-create table", e); - table = null; + if (mvWork.getLoadFileWork().getCtasCreateTableDesc() != null) { + try { + table = mvWork.getLoadFileWork().getCtasCreateTableDesc().toTable(hconf); + } catch (HiveException e) { + LOG.debug("can't pre-create table for CTAS", e); + table = null; + } + } else if (mvWork.getLoadFileWork().getCreateViewDesc() != null) { + if (mvWork.getLoadFileWork().getCreateViewDesc().isReplace()) { + // ALTER MV ... REBUILD + String tableName = mvWork.getLoadFileWork().getCreateViewDesc().getViewName(); + try { + table = Hive.get().getTable(tableName); + } catch (HiveException e) { + throw new RuntimeException("unexpected; MV should be present already..: " + tableName, e); + } + } else { + // CREATE MATERIALIZED VIEW ... + try { + table = mvWork.getLoadFileWork().getCreateViewDesc().toTable(hconf); + } catch (HiveException e) { + LOG.debug("can't pre-create table for MV", e); + table = null; + } + } + } else { + throw new RuntimeException("unexpected; this should be a CTAS or a CREATE/REBUILD MV - however no desc present"); } } assert statsWork != null : "Error when generating StatsTask"; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index e9ae590f4c..d36d24d090 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1164,7 +1164,9 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION || ast.getToken().getType() == HiveParser.TOK_TABTYPE || ast.getToken().getType() == HiveParser.TOK_CREATETABLE - || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW); + || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW + || (ast.getToken().getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW && + ast.getChild(1).getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD)); int childIndex = 0; numDynParts = 0; @@ -1177,7 +1179,8 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit + tableName; } if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE && - ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW) { + ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW && + ast.getToken().getType() != HiveParser.TOK_ALTER_MATERIALIZED_VIEW) { tableHandle = db.getTable(tableName); } } catch (InvalidTableException ite) { @@ -1190,7 +1193,8 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit // get partition metadata if partition specified if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE && - ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW) { + ast.getToken().getType() != HiveParser.TOK_CREATE_MATERIALIZED_VIEW && + ast.getToken().getType() != HiveParser.TOK_ALTER_MATERIALIZED_VIEW) { childIndex = 1; ASTNode partspec = (ASTNode) ast.getChild(1); partitions = new ArrayList(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 55b1da9030..76c82e2606 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -370,6 +370,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept disableJoinMerge = true; boolean reAnalyzeAST = false; final boolean materializedView = getQB().isMaterializedView(); + final boolean rebuild = materializedView && createVwDesc.isReplace(); try { if (this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { @@ -389,7 +390,10 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept ASTNode newAST = getOptimizedAST(); // 1.1. Fix up the query for insert/ctas/materialized views - newAST = fixUpAfterCbo(ast, newAST, cboCtx); + if (!rebuild) { + // If it is not a MATERIALIZED VIEW...REBUILD + newAST = fixUpAfterCbo(ast, newAST, cboCtx); + } // 2. Regen OP plan from optimized AST if (cboCtx.type == PreCboCtx.Type.VIEW && !materializedView) { @@ -399,25 +403,32 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept throw new CalciteViewSemanticException(e.getMessage()); } } else if (cboCtx.type == PreCboCtx.Type.VIEW && materializedView) { - // Store text of the ORIGINAL QUERY - String originalText = ctx.getTokenRewriteStream().toString( - cboCtx.nodeOfInterest.getTokenStartIndex(), - cboCtx.nodeOfInterest.getTokenStopIndex()); - unparseTranslator.applyTranslations(ctx.getTokenRewriteStream()); - String expandedText = ctx.getTokenRewriteStream().toString( - cboCtx.nodeOfInterest.getTokenStartIndex(), - cboCtx.nodeOfInterest.getTokenStopIndex()); - // Redo create-table/view analysis, because it's not part of - // doPhase1. - // Use the REWRITTEN AST - init(false); - setAST(newAST); - newAST = reAnalyzeViewAfterCbo(newAST); + if (rebuild) { + // Use the CREATE MATERIALIZED VIEW...REBUILD + init(false); + setAST(ast); + reAnalyzeViewAfterCbo(ast); + } else { + // Store text of the ORIGINAL QUERY + String originalText = ctx.getTokenRewriteStream().toString( + cboCtx.nodeOfInterest.getTokenStartIndex(), + cboCtx.nodeOfInterest.getTokenStopIndex()); + unparseTranslator.applyTranslations(ctx.getTokenRewriteStream()); + String expandedText = ctx.getTokenRewriteStream().toString( + cboCtx.nodeOfInterest.getTokenStartIndex(), + cboCtx.nodeOfInterest.getTokenStopIndex()); + // Redo create-table/view analysis, because it's not part of + // doPhase1. + // Use the REWRITTEN AST + init(false); + setAST(newAST); + newAST = reAnalyzeViewAfterCbo(newAST); + createVwDesc.setViewOriginalText(originalText); + createVwDesc.setViewExpandedText(expandedText); + } viewSelect = newAST; viewsExpanded = new ArrayList<>(); viewsExpanded.add(createVwDesc.getViewName()); - createVwDesc.setViewOriginalText(originalText); - createVwDesc.setViewExpandedText(expandedText); } else if (cboCtx.type == PreCboCtx.Type.CTAS) { // CTAS init(false); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index f3f1a7e8da..f90bb8acac 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -262,6 +262,7 @@ TOK_CREATE_MATERIALIZED_VIEW; TOK_DROP_MATERIALIZED_VIEW; TOK_ALTER_MATERIALIZED_VIEW; TOK_ALTER_MATERIALIZED_VIEW_REWRITE; +TOK_ALTER_MATERIALIZED_VIEW_REBUILD; TOK_REWRITE_ENABLED; TOK_REWRITE_DISABLED; TOK_VIEWPARTCOLS; @@ -1366,6 +1367,7 @@ alterMaterializedViewStatementSuffix @init { pushMsg("alter materialized view statement", state); } @after { popMsg(state); } : alterMaterializedViewSuffixRewrite + | alterMaterializedViewSuffixRebuild ; alterIndexStatementSuffix @@ -1542,6 +1544,12 @@ alterMaterializedViewSuffixRewrite -> ^(TOK_ALTER_MATERIALIZED_VIEW_REWRITE $mvRewriteFlag) ; +alterMaterializedViewSuffixRebuild +@init { pushMsg("alter materialized view rebuild statement", state); } +@after { popMsg(state); } + : KW_REBUILD -> ^(TOK_ALTER_MATERIALIZED_VIEW_REBUILD) + ; + alterStatementSuffixSerdeProperties @init { pushMsg("alter serdes statement", state); } @after { popMsg(state); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1de3dd7230..37d36d8254 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -11364,6 +11364,8 @@ boolean genResolvedParseTree(ASTNode ast, PlannerContext plannerCtx) throws Sema // 3. analyze create view command if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW || ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW || + (ast.getToken().getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW && + ast.getChild(1).getType() == HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD) || (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) { child = analyzeCreateView(ast, qb, plannerCtx); @@ -11749,6 +11751,11 @@ private void putAccessedColumnsToReadEntity(HashSet inputs, ColumnAc } protected void saveViewDefinition() throws SemanticException { + if (createVwDesc.isMaterialized() && createVwDesc.isReplace()) { + // This is a rebuild, there's nothing to do here. + return; + } + // Make a copy of the statement's result schema, since we may // modify it below as part of imposing view column names. List derivedSchema = @@ -12593,6 +12600,7 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt Map tblProps = null; List partColNames = null; boolean isMaterialized = ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW; + boolean isRebuild = false; String location = null; RowFormatParams rowFormatParams = new RowFormatParams(); StorageFormat storageFormat = new StorageFormat(conf); @@ -12615,6 +12623,10 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt case HiveParser.TOK_ORREPLACE: orReplace = true; break; + case HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD: + isMaterialized = true; + isRebuild = true; + break; case HiveParser.TOK_QUERY: // For CBO if (plannerCtx != null) { @@ -12672,7 +12684,7 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt if (isMaterialized) { createVwDesc = new CreateViewDesc( dbDotTable, cols, comment, tblProps, partColNames, - ifNotExists, orReplace, rewriteEnabled, isAlterViewAs, + ifNotExists, orReplace || isRebuild, rewriteEnabled, isAlterViewAs, storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, storageFormat.getSerde(), storageFormat.getStorageHandler(), storageFormat.getSerdeProps()); @@ -12690,6 +12702,25 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt } qb.setViewDesc(createVwDesc); + if (isRebuild) { + // We need to go lookup the table and get the select statement and then parse it. + try { + Table tab = getTableObjectByName(dbDotTable, true); + String viewText = tab.getViewOriginalText(); + if (viewText.trim().isEmpty()) { + throw new SemanticException(ErrorMsg.MATERIALIZED_VIEW_DEF_EMPTY); + } + Context ctx = new Context(queryState.getConf()); + selectStmt = ParseUtils.parse(viewText, ctx); + // For CBO + if (plannerCtx != null) { + plannerCtx.setViewToken(selectStmt); + } + } catch (Exception e) { + throw new SemanticException(e); + } + } + return selectStmt; } @@ -12726,7 +12757,7 @@ private void validateCreateView() } //replace view - if (createVwDesc.getOrReplace() && oldView != null) { + if (createVwDesc.isReplace() && oldView != null) { // Don't allow swapping between virtual and materialized view in replace if (oldView.getTableType().equals(TableType.VIRTUAL_VIEW) && createVwDesc.isMaterialized()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java index d139155a1d..a3b3287663 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java @@ -273,6 +273,11 @@ private static BaseSemanticAnalyzer getInternal(QueryState queryState, ASTNode t opType = commandType.get(child.getType()); queryState.setCommandType(opType); return new DDLSemanticAnalyzer(queryState); + case HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD: + opType = commandType.get(child.getType()); + queryState.setCommandType(opType); + return HiveConf.getBoolVar(queryState.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED) ? + new CalcitePlanner(queryState) : new SemanticAnalyzer(queryState); } // Operation not recognized, set to null and let upper level handle this case queryState.setCommandType(null); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java index 0621bd413e..26bb3e1707 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java @@ -179,6 +179,11 @@ public boolean isTargetRewritten() { if (getLoadFileDesc() != null && getLoadFileDesc().getCtasCreateTableDesc() != null) { return true; } + // ALTER MV ... REBUILD + if (getLoadFileDesc() != null && getLoadFileDesc().getCreateViewDesc() != null && + getLoadFileDesc().getCreateViewDesc().isReplace()) { + return true; + } return false; } @@ -188,8 +193,10 @@ public String getTableName() { return work.getLoadTableDesc().getTable().getTableName(); } else if (work.getTableSpecs() != null) { return work.getTableSpecs().tableName; - } else { + } else if (getLoadFileDesc().getCtasCreateTableDesc() != null) { return getLoadFileDesc().getCtasCreateTableDesc().getTableName(); + } else { + return getLoadFileDesc().getCreateViewDesc().getViewName(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java index 9425f6e891..09aa82f1f0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java @@ -22,10 +22,20 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.DDLTask; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.Explain.Level; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -35,6 +45,7 @@ @Explain(displayName = "Create View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public class CreateViewDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = 1L; + private static Logger LOG = LoggerFactory.getLogger(CreateViewDesc.class); private String viewName; private String originalText; @@ -46,7 +57,7 @@ private List partCols; private String comment; private boolean ifNotExists; - private boolean orReplace; + private boolean replace; private boolean isAlterViewAs; private boolean isMaterialized; private String inputFormat; @@ -83,7 +94,7 @@ public CreateViewDesc() { */ public CreateViewDesc(String viewName, List schema, String comment, Map tblProps, List partColNames, - boolean ifNotExists, boolean orReplace, boolean rewriteEnabled, boolean isAlterViewAs, + boolean ifNotExists, boolean replace, boolean rewriteEnabled, boolean isAlterViewAs, String inputFormat, String outputFormat, String location, String serde, String storageHandler, Map serdeProps) { this.viewName = viewName; @@ -92,7 +103,7 @@ public CreateViewDesc(String viewName, List schema, String comment, this.partColNames = partColNames; this.comment = comment; this.ifNotExists = ifNotExists; - this.orReplace = orReplace; + this.replace = replace; this.isMaterialized = true; this.rewriteEnabled = rewriteEnabled; this.isAlterViewAs = isAlterViewAs; @@ -128,7 +139,7 @@ public CreateViewDesc(String viewName, List schema, String comment, this.partColNames = partColNames; this.comment = comment; this.ifNotExists = ifNotExists; - this.orReplace = orReplace; + this.replace = orReplace; this.isAlterViewAs = isAlterViewAs; this.isMaterialized = false; this.rewriteEnabled = false; @@ -164,7 +175,7 @@ public void setViewExpandedText(String expandedText) { this.expandedText = expandedText; } - @Explain(displayName = "rewrite enabled") + @Explain(displayName = "rewrite enabled", displayOnlyOnTrue = true) public boolean isRewriteEnabled() { return rewriteEnabled; } @@ -234,13 +245,13 @@ public void setIfNotExists(boolean ifNotExists) { this.ifNotExists = ifNotExists; } - @Explain(displayName = "or replace") - public boolean getOrReplace() { - return orReplace; + @Explain(displayName = "replace", displayOnlyOnTrue = true) + public boolean isReplace() { + return replace; } - public void setOrReplace(boolean orReplace) { - this.orReplace = orReplace; + public void setReplace(boolean replace) { + this.replace = replace; } @Explain(displayName = "is alter view as select", displayOnlyOnTrue = true) @@ -308,4 +319,88 @@ public ReplicationSpec getReplicationSpec(){ } return this.replicationSpec; } + + public Table toTable(HiveConf conf) throws HiveException { + String[] names = Utilities.getDbTableName(getViewName()); + String databaseName = names[0]; + String tableName = names[1]; + + Table tbl = new Table(databaseName, tableName); + tbl.setViewOriginalText(getViewOriginalText()); + tbl.setViewExpandedText(getViewExpandedText()); + if (isMaterialized()) { + tbl.setRewriteEnabled(isRewriteEnabled()); + tbl.setTableType(TableType.MATERIALIZED_VIEW); + } else { + tbl.setTableType(TableType.VIRTUAL_VIEW); + } + tbl.setSerializationLib(null); + tbl.clearSerDeInfo(); + tbl.setFields(getSchema()); + if (getComment() != null) { + tbl.setProperty("comment", getComment()); + } + if (getTblProps() != null) { + tbl.getTTable().getParameters().putAll(getTblProps()); + } + + if (getPartCols() != null) { + tbl.setPartCols(getPartCols()); + } + + if (getInputFormat() != null) { + tbl.setInputFormatClass(getInputFormat()); + } + + if (getOutputFormat() != null) { + tbl.setOutputFormatClass(getOutputFormat()); + } + + if (isMaterialized()) { + if (getLocation() != null) { + tbl.setDataLocation(new Path(getLocation())); + } + + if (getStorageHandler() != null) { + tbl.setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, + getStorageHandler()); + } + HiveStorageHandler storageHandler = tbl.getStorageHandler(); + + /* + * If the user didn't specify a SerDe, we use the default. + */ + String serDeClassName; + if (getSerde() == null) { + if (storageHandler == null) { + serDeClassName = PlanUtils.getDefaultSerDe().getName(); + LOG.info("Default to {} for materialized view {}", serDeClassName, + getViewName()); + } else { + serDeClassName = storageHandler.getSerDeClass().getName(); + LOG.info("Use StorageHandler-supplied {} for materialized view {}", + serDeClassName, getViewName()); + } + } else { + // let's validate that the serde exists + serDeClassName = getSerde(); + DDLTask.validateSerDe(serDeClassName, conf); + } + tbl.setSerializationLib(serDeClassName); + + // To remain consistent, we need to set input and output formats both + // at the table level and the storage handler level. + tbl.setInputFormatClass(getInputFormat()); + tbl.setOutputFormatClass(getOutputFormat()); + if (getInputFormat() != null && !getInputFormat().isEmpty()) { + tbl.getSd().setInputFormat(tbl.getInputFormatClass().getName()); + } + if (getOutputFormat() != null && !getOutputFormat().isEmpty()) { + tbl.getSd().setOutputFormat(tbl.getOutputFormatClass().getName()); + } + } + + return tbl; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java index 1770046869..33e30bf10f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java @@ -303,7 +303,7 @@ public void setReplaceMode(boolean replaceMode) { createTblDesc.setReplaceMode(replaceMode); break; case VIEW: - createViewDesc.setOrReplace(replaceMode); + createViewDesc.setReplace(replaceMode); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java index c09589c5f2..c3dab35e6f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java @@ -36,6 +36,7 @@ private String columns; private String columnTypes; private transient CreateTableDesc ctasCreateTableDesc; + private transient CreateViewDesc createViewDesc; private boolean isMmCtas; public LoadFileDesc(final LoadFileDesc o) { @@ -47,21 +48,26 @@ public LoadFileDesc(final LoadFileDesc o) { this.columnTypes = o.columnTypes; this.isMmCtas = o.isMmCtas; this.ctasCreateTableDesc = o.ctasCreateTableDesc; + this.createViewDesc = o.createViewDesc; } - public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc, - final Path sourcePath, final Path targetDir, final boolean isDfsDir, + public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc, + final Path sourcePath, final Path targetDir, final boolean isDfsDir, final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) { this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas); - if (createTableDesc != null && createTableDesc.isCTAS()) { - ctasCreateTableDesc = createTableDesc; + if (createTableDesc != null && createTableDesc.isCTAS()) { + this.ctasCreateTableDesc = createTableDesc; + } + if (createViewDesc != null && createViewDesc.isMaterialized()) { + this.createViewDesc = createViewDesc; } } public LoadFileDesc(final Path sourcePath, final Path targetDir, - final boolean isDfsDir, final String columns, final String columnTypes, boolean isMmCtas) { + final boolean isDfsDir, final String columns, final String columnTypes, boolean isMmCtas) { this(sourcePath, targetDir, isDfsDir, columns, columnTypes, AcidUtils.Operation.NOT_ACID, isMmCtas); } + private LoadFileDesc(final Path sourcePath, final Path targetDir, final boolean isDfsDir, final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) { @@ -128,6 +134,10 @@ public CreateTableDesc getCtasCreateTableDesc() { return ctasCreateTableDesc; } + public CreateViewDesc getCreateViewDesc() { + return createViewDesc; + } + public boolean isMmCtas() { return isMmCtas; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index 3c1e92af12..81cc27918d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -442,6 +442,11 @@ public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, St crtViewDesc.getStorageHandler()); } + if (crtViewDesc.getViewName() != null && crtViewDesc.isMaterialized()) { + properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, + crtViewDesc.getViewName()); + } + if (crtViewDesc.getTblProps() != null) { properties.putAll(crtViewDesc.getTblProps()); } diff --git ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q new file mode 100644 index 0000000000..a2e7d38215 --- /dev/null +++ ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_no_grant.q @@ -0,0 +1,20 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvrng_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +grant select on table amvrng_table to user user2 with grant option; + +set user.name=user2; +create materialized view amvrng_mat_view as select a, c from amvrng_table; + +set user.name=user1; +revoke grant option for select on table amvrng_table from user user2; + +set user.name=user2; +alter materialized view amvrng_mat_view rebuild; diff --git ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q new file mode 100644 index 0000000000..7c2d145ada --- /dev/null +++ ql/src/test/queries/clientnegative/materialized_view_authorization_rebuild_other.q @@ -0,0 +1,14 @@ +set hive.test.authz.sstd.hs2.mode=true; +set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest; +set hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator; +set hive.security.authorization.enabled=true; +set user.name=user1; + +create table amvro_table (a int, b varchar(256), c decimal(10,2)); + +insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8); + +create materialized view amvro_mat_view as select a, c from amvro_table; + +set user.name=user2; +alter materialized view amvro_mat_view rebuild; diff --git ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q index 1749cb023a..761903fd58 100644 --- ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q +++ ql/src/test/queries/clientpositive/materialized_view_create_rewrite.q @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + set hive.strict.checks.cartesian.product=false; set hive.materializedview.rewriting=true; set hive.stats.column.autogather=true; diff --git ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q new file mode 100644 index 0000000000..6462d9a677 --- /dev/null +++ ql/src/test/queries/clientpositive/materialized_view_create_rewrite_3.q @@ -0,0 +1,78 @@ +set hive.strict.checks.cartesian.product=false; +set hive.materializedview.rewriting=true; +set hive.stats.column.autogather=true; + +create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int); + +insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1); + +create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int); + +insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3); + +EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- USE THE VIEW +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1); + +-- TODO: CANNOT USE THE VIEW, IT IS OUTDATED +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +-- REBUILD +EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD; + +-- NOW IT CAN BE USED AGAIN +EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c; + +drop materialized view cmv_mat_view; diff --git ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out new file mode 100644 index 0000000000..28f0a72b41 --- /dev/null +++ ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_no_grant.q.out @@ -0,0 +1,40 @@ +PREHOOK: query: create table amvrng_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: create table amvrng_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvrng_table +PREHOOK: query: insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: insert into amvrng_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@amvrng_table +POSTHOOK: Lineage: amvrng_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvrng_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvrng_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: grant select on table amvrng_table to user user2 with grant option +PREHOOK: type: GRANT_PRIVILEGE +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: grant select on table amvrng_table to user user2 with grant option +POSTHOOK: type: GRANT_PRIVILEGE +POSTHOOK: Output: default@amvrng_table +PREHOOK: query: create materialized view amvrng_mat_view as select a, c from amvrng_table +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@amvrng_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvrng_mat_view +POSTHOOK: query: create materialized view amvrng_mat_view as select a, c from amvrng_table +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@amvrng_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvrng_mat_view +PREHOOK: query: revoke grant option for select on table amvrng_table from user user2 +PREHOOK: type: REVOKE_PRIVILEGE +PREHOOK: Output: default@amvrng_table +POSTHOOK: query: revoke grant option for select on table amvrng_table from user user2 +POSTHOOK: type: REVOKE_PRIVILEGE +POSTHOOK: Output: default@amvrng_table +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvrng_table]] diff --git ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out new file mode 100644 index 0000000000..72244aaec7 --- /dev/null +++ ql/src/test/results/clientnegative/materialized_view_authorization_rebuild_other.q.out @@ -0,0 +1,28 @@ +PREHOOK: query: create table amvro_table (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@amvro_table +POSTHOOK: query: create table amvro_table (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvro_table +PREHOOK: query: insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Output: default@amvro_table +POSTHOOK: query: insert into amvro_table values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@amvro_table +POSTHOOK: Lineage: amvro_table.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: amvro_table.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: amvro_table.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view amvro_mat_view as select a, c from amvro_table +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@amvro_table +PREHOOK: Output: database:default +PREHOOK: Output: default@amvro_mat_view +POSTHOOK: query: create materialized view amvro_mat_view as select a, c from amvro_table +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@amvro_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@amvro_mat_view +FAILED: HiveAccessControlException Permission denied: Principal [name=user2, type=USER] does not have following privileges for operation CREATE_MATERIALIZED_VIEW [[SELECT with grant] on Object [type=TABLE_OR_VIEW, name=default.amvro_table]] diff --git ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out index 81a7950773..aa3240cad4 100644 --- ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out +++ ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out @@ -52,6 +52,7 @@ PREHOOK: query: show tblproperties cmv_mat_view PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} numFiles 1 totalSize 453 #### A masked pattern was here #### @@ -75,12 +76,13 @@ POSTHOOK: query: select * from cmv_mat_view2 POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_mat_view2 #### A masked pattern was here #### -3 978.76 3 9.80 +3 978.76 PREHOOK: query: show tblproperties cmv_mat_view2 PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} numFiles 1 totalSize 322 #### A masked pattern was here #### @@ -117,8 +119,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_mat_view2 #### A masked pattern was here #### -3 978.76 3 9.80 +3 978.76 PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE PREHOOK: Input: default@cmv_mat_view2 diff --git ql/src/test/results/clientpositive/create_view.q.out ql/src/test/results/clientpositive/create_view.q.out index 8763fdb75f..ffe69eb8c7 100644 --- ql/src/test/results/clientpositive/create_view.q.out +++ ql/src/test/results/clientpositive/create_view.q.out @@ -168,12 +168,10 @@ STAGE PLANS: Stage: Stage-1 Create View Operator: Create View - or replace: false columns: valoo string expanded text: SELECT `_c0` AS `valoo` FROM (SELECT upper(`src`.`value`) FROM `default`.`src` WHERE `src`.`key`=86) `default.view0` name: default.view0 original text: SELECT upper(value) FROM src WHERE key=86 - rewrite enabled: false PREHOOK: query: EXPLAIN SELECT * from view2 where key=18 diff --git ql/src/test/results/clientpositive/create_view_translate.q.out ql/src/test/results/clientpositive/create_view_translate.q.out index e5748976bd..2365a7409e 100644 --- ql/src/test/results/clientpositive/create_view_translate.q.out +++ ql/src/test/results/clientpositive/create_view_translate.q.out @@ -135,12 +135,10 @@ STAGE PLANS: Stage: Stage-1 Create View Operator: Create View - or replace: false columns: id int, _c1 string expanded text: SELECT `items`.`id`, `items`.`info`['price'] FROM `default`.`items` name: default.priceview original text: SELECT items.id, items.info['price'] FROM items - rewrite enabled: false PREHOOK: query: CREATE VIEW priceview AS SELECT items.id, items.info['price'] FROM items PREHOOK: type: CREATEVIEW diff --git ql/src/test/results/clientpositive/explain_ddl.q.out ql/src/test/results/clientpositive/explain_ddl.q.out index f0e54c5153..8a17a66925 100644 --- ql/src/test/results/clientpositive/explain_ddl.q.out +++ ql/src/test/results/clientpositive/explain_ddl.q.out @@ -427,12 +427,10 @@ STAGE PLANS: Stage: Stage-1 Create View Operator: Create View - or replace: false columns: key string, value string expanded text: select `m1`.`key`, `m1`.`value` from `default`.`M1` name: default.V1 original text: select * from M1 - rewrite enabled: false PREHOOK: query: EXPLAIN CREATE TABLE M1 LIKE src PREHOOK: type: CREATETABLE diff --git ql/src/test/results/clientpositive/llap/materialized_view_create.q.out ql/src/test/results/clientpositive/llap/materialized_view_create.q.out new file mode 100644 index 0000000000..928618390d --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_create.q.out @@ -0,0 +1,207 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view as select a, b, c from cmv_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view as select a, b, c from cmv_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: select * from cmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 alfred 10.30 +2 bob 3.14 +2 bonnie 172342.20 +3 calvin 978.76 +3 charlie 9.80 +PREHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: select * from cmv_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +1 10.30 +2 3.14 +2 172342.20 +3 978.76 +3 9.80 +PREHOOK: query: create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view3 +POSTHOOK: query: create materialized view if not exists cmv_mat_view3 as select * from cmv_basetable where a > 1 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view3 +PREHOOK: query: select * from cmv_mat_view3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +2 bob 3.14 +2 bonnie 172342.20 +3 calvin 978.76 +3 charlie 9.80 +PREHOOK: query: create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view4 +POSTHOOK: query: create materialized view cmv_mat_view4 comment 'this is a comment' as select a, sum(c) from cmv_basetable group by a +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view4 +PREHOOK: query: select * from cmv_mat_view4 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view4 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view4 +#### A masked pattern was here #### +1 10.30 +2 172345.34 +3 988.56 +PREHOOK: query: describe extended cmv_mat_view4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view4 +POSTHOOK: query: describe extended cmv_mat_view4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view4 +a int +_c1 decimal(20,2) + +#### A masked pattern was here #### +PREHOOK: query: create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable2 +POSTHOOK: query: create table cmv_basetable2 (d int, e varchar(256), f decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable2 +PREHOOK: query: insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable2 +POSTHOOK: query: insert into cmv_basetable2 values (4, 'alfred', 100.30),(4, 'bob', 6133,14),(5, 'bonnie', 172.2),(6, 'calvin', 8.76),(17, 'charlie', 13144339.8) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable2 +POSTHOOK: Lineage: cmv_basetable2.d EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable2.e EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable2.f EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view5 +POSTHOOK: query: create materialized view cmv_mat_view5 tblproperties ('key'='value') as select a, b, d, c, f from cmv_basetable t1 join cmv_basetable2 t2 on (t1.b = t2.e) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view5 +PREHOOK: query: select * from cmv_mat_view5 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view5 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view5 +#### A masked pattern was here #### +1 alfred 4 10.30 100.30 +2 bob 4 3.14 6133.00 +2 bonnie 5 172342.20 172.20 +3 calvin 6 978.76 8.76 +3 charlie 17 9.80 13144339.80 +PREHOOK: query: show tblproperties cmv_mat_view5 +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view5 +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} +key value +numFiles 1 +totalSize 710 +#### A masked pattern was here #### +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: drop materialized view cmv_mat_view2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: drop materialized view cmv_mat_view2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: drop materialized view cmv_mat_view3 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view3 +PREHOOK: Output: default@cmv_mat_view3 +POSTHOOK: query: drop materialized view cmv_mat_view3 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view3 +POSTHOOK: Output: default@cmv_mat_view3 +PREHOOK: query: drop materialized view cmv_mat_view4 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view4 +PREHOOK: Output: default@cmv_mat_view4 +POSTHOOK: query: drop materialized view cmv_mat_view4 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view4 +POSTHOOK: Output: default@cmv_mat_view4 +PREHOOK: query: drop materialized view cmv_mat_view5 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view5 +PREHOOK: Output: default@cmv_mat_view5 +POSTHOOK: query: drop materialized view cmv_mat_view5 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view5 +POSTHOOK: Output: default@cmv_mat_view5 diff --git ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out new file mode 100644 index 0000000000..8bebab4ef0 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out @@ -0,0 +1,501 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view enable rewrite +as select a, b, c from cmv_basetable where a = 2 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view enable rewrite +as select a, b, c from cmv_basetable where a = 2 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: select * from cmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +2 bob 3.14 +2 bonnie 172342.20 +PREHOOK: query: show tblproperties cmv_mat_view +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} +numFiles 1 +totalSize 453 +#### A masked pattern was here #### +PREHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from cmv_basetable where a = 3 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from cmv_basetable where a = 3 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: select * from cmv_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +3 9.80 +3 978.76 +PREHOOK: query: show tblproperties cmv_mat_view2 +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view2 +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} +numFiles 1 +totalSize 322 +#### A masked pattern was here #### +PREHOOK: query: explain +select a, c from cmv_basetable where a = 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a, c from cmv_basetable where a = 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view2 + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: select a, c from cmv_basetable where a = 3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select a, c from cmv_basetable where a = 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +3 9.80 +3 978.76 +PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: alter materialized view cmv_mat_view2 disable rewrite +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (a = 3) (type: boolean) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((3 = a) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +3 9.80 3 978.76 +3 978.76 3 978.76 +PREHOOK: query: explain +alter materialized view cmv_mat_view2 enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: query: explain +alter materialized view cmv_mat_view2 enable rewrite +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Alter Materialized View Operator: + Alter Materialized View + name: default.cmv_mat_view2 + operation: UPDATE_REWRITE_FLAG + +PREHOOK: query: alter materialized view cmv_mat_view2 enable rewrite +PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite +POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view2 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((3 = a) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +3 9.80 3 978.76 +3 978.76 3 978.76 +PREHOOK: query: drop materialized view cmv_mat_view2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: drop materialized view cmv_mat_view2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +POSTHOOK: query: explain +select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (a = 3) (type: boolean) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((3 = a) and (d = 3)) (type: boolean) + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: c (type: decimal(10,2)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 464 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: select * from ( + (select a, c from cmv_basetable where a = 3) table1 + join + (select a, c from cmv_basetable where d = 3) table2 + on table1.a = table2.a) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +3 9.80 3 978.76 +3 978.76 3 978.76 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view diff --git ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out new file mode 100644 index 0000000000..83ab7429e4 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out @@ -0,0 +1,714 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view enable rewrite +as select b from cmv_basetable where c > 10.0 group by a, b, c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view enable rewrite +as select b from cmv_basetable where c > 10.0 group by a, b, c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: explain +select b from cmv_basetable where c > 20.0 group by a, b +PREHOOK: type: QUERY +POSTHOOK: query: explain +select b from cmv_basetable where c > 20.0 group by a, b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 1030 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (c > 20) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), b (type: varchar(256)) + outputColumnNames: a, b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: a (type: int), b (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: varchar(256)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 90 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 90 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +bonnie +calvin +PREHOOK: query: create materialized view cmv_mat_view_2 enable rewrite +as select b, c from cmv_basetable where c > 10.0 group by a, b, c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view_2 +POSTHOOK: query: create materialized view cmv_mat_view_2 enable rewrite +as select b, c from cmv_basetable where c > 10.0 group by a, b, c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view_2 +PREHOOK: query: explain +select b from cmv_basetable where c > 20.0 group by a, b +PREHOOK: type: QUERY +POSTHOOK: query: explain +select b from cmv_basetable where c > 20.0 group by a, b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 1030 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (c > 20) (type: boolean) + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), b (type: varchar(256)) + outputColumnNames: a, b + Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: a (type: int), b (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: varchar(256)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 90 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 90 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +bonnie +calvin +PREHOOK: query: create materialized view cmv_mat_view_3 enable rewrite +as select a, b, c from cmv_basetable where c > 10.0 group by a, b, c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view_3 +POSTHOOK: query: create materialized view cmv_mat_view_3 enable rewrite +as select a, b, c from cmv_basetable where c > 10.0 group by a, b, c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view_3 +PREHOOK: query: explain +select b from cmv_basetable where c > 20.0 group by a, b +PREHOOK: type: QUERY +POSTHOOK: query: explain +select b from cmv_basetable where c > 20.0 group by a, b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view_3 + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (c > 20) (type: boolean) + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), b (type: varchar(256)) + outputColumnNames: a, b + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: a (type: int), b (type: varchar(256)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: varchar(256)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: varchar(256)) + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: varchar(256)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 456 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_mat_view_3 +#### A masked pattern was here #### +POSTHOOK: query: select b from cmv_basetable where c > 20.0 group by a, b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_mat_view_3 +#### A masked pattern was here #### +bonnie +calvin +PREHOOK: query: create materialized view cmv_mat_view_4 enable rewrite +as select a, b from cmv_basetable group by a, b +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view_4 +POSTHOOK: query: create materialized view cmv_mat_view_4 enable rewrite +as select a, b from cmv_basetable group by a, b +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view_4 +PREHOOK: query: explain +select b from cmv_basetable group by b +PREHOOK: type: QUERY +POSTHOOK: query: explain +select b from cmv_basetable group by b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view_4 + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: b (type: varchar(256)) + outputColumnNames: b + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: b (type: varchar(256)) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: varchar(256)) + sort order: + + Map-reduce partition columns: _col0 (type: varchar(256)) + Statistics: Num rows: 2 Data size: 680 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: varchar(256)) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select b from cmv_basetable group by b +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_mat_view_4 +#### A masked pattern was here #### +POSTHOOK: query: select b from cmv_basetable group by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_mat_view_4 +#### A masked pattern was here #### +bob +alfred +bonnie +charlie +calvin +PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.b EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.c EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.d EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view_5 enable rewrite +as select cmv_basetable.a, cmv_basetable_2.c + from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) + where cmv_basetable_2.c > 10.0 + group by cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view_5 +POSTHOOK: query: create materialized view cmv_mat_view_5 enable rewrite +as select cmv_basetable.a, cmv_basetable_2.c + from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) + where cmv_basetable_2.c > 10.0 + group by cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view_5 +PREHOOK: query: explain +select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view_4 + Statistics: Num rows: 159 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: a (type: int) + sort order: + + Map-reduce partition columns: a (type: int) + Statistics: Num rows: 159 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: a (type: int) + sort order: + + Map-reduce partition columns: a (type: int) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: c (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 a (type: int) + 1 a (type: int) + outputColumnNames: _col0, _col5, _col7 + Statistics: Num rows: 174 Data size: 673 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col5) and (_col7 > 10.1)) (type: boolean) + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col7 (type: decimal(10,2)) + outputColumnNames: _col0, _col7 + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col7 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 14 Data size: 54 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 14 Data size: 54 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 14 Data size: 54 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view_4 +#### A masked pattern was here #### +POSTHOOK: query: select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view_4 +#### A masked pattern was here #### +3 +1 +PREHOOK: query: explain +select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a +PREHOOK: type: QUERY +POSTHOOK: query: explain +select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.cmv_mat_view_4 + Statistics: Num rows: 159 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: a (type: int) + sort order: + + Map-reduce partition columns: a (type: int) + Statistics: Num rows: 159 Data size: 612 Basic stats: COMPLETE Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: a (type: int) + sort order: + + Map-reduce partition columns: a (type: int) + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: c (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 a (type: int) + 1 a (type: int) + outputColumnNames: _col0, _col5, _col7 + Statistics: Num rows: 174 Data size: 673 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = _col5) and (_col7 > 10.1)) (type: boolean) + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 29 Data size: 112 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 14 Data size: 54 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 14 Data size: 54 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view_4 +#### A masked pattern was here #### +POSTHOOK: query: select cmv_basetable.a +from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) +where cmv_basetable_2.c > 10.10 +group by cmv_basetable.a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view_4 +#### A masked pattern was here #### +1 +3 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: drop materialized view cmv_mat_view_2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view_2 +PREHOOK: Output: default@cmv_mat_view_2 +POSTHOOK: query: drop materialized view cmv_mat_view_2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view_2 +POSTHOOK: Output: default@cmv_mat_view_2 +PREHOOK: query: drop materialized view cmv_mat_view_3 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view_3 +PREHOOK: Output: default@cmv_mat_view_3 +POSTHOOK: query: drop materialized view cmv_mat_view_3 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view_3 +POSTHOOK: Output: default@cmv_mat_view_3 +PREHOOK: query: drop materialized view cmv_mat_view_4 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view_4 +PREHOOK: Output: default@cmv_mat_view_4 +POSTHOOK: query: drop materialized view cmv_mat_view_4 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view_4 +POSTHOOK: Output: default@cmv_mat_view_4 +PREHOOK: query: drop materialized view cmv_mat_view_5 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view_5 +PREHOOK: Output: default@cmv_mat_view_5 +POSTHOOK: query: drop materialized view cmv_mat_view_5 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view_5 +POSTHOOK: Output: default@cmv_mat_view_5 diff --git ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out new file mode 100644 index 0000000000..58ba9ff781 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out @@ -0,0 +1,504 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.b EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.c EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.d EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View Operator: + Create View + columns: a int, c decimal(10,2) + expanded text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 + GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` + name: default.cmv_mat_view + original text: SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c + rewrite enabled: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 +1 +PREHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.c EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.d EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 +1 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2, Stage-0 + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 20 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(10,2)) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View Operator: + Create View + columns: a int, c decimal(10,2) + name: default.cmv_mat_view + replace: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: default.cmv_mat_view + Filter Operator + predicate: (c > 10.1) (type: boolean) + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +3 +3 +1 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view diff --git ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out new file mode 100644 index 0000000000..e135785363 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out @@ -0,0 +1,157 @@ +PREHOOK: query: create database db1 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:db1 +POSTHOOK: query: create database db1 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:db1 +PREHOOK: query: use db1 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:db1 +POSTHOOK: query: use db1 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:db1 +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:db1 +PREHOOK: Output: db1@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:db1 +POSTHOOK: Output: db1@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: db1@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: db1@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: create database db2 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:db2 +POSTHOOK: query: create database db2 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:db2 +PREHOOK: query: use db2 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:db2 +POSTHOOK: query: use db2 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:db2 +PREHOOK: query: create materialized view cmv_mat_view enable rewrite +as select a, b, c from db1.cmv_basetable where a = 2 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: db1@cmv_basetable +PREHOOK: Output: database:db2 +PREHOOK: Output: db2@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view enable rewrite +as select a, b, c from db1.cmv_basetable where a = 2 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: db1@cmv_basetable +POSTHOOK: Output: database:db2 +POSTHOOK: Output: db2@cmv_mat_view +PREHOOK: query: select * from cmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: db2@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: db2@cmv_mat_view +#### A masked pattern was here #### +2 bob 3.14 +2 bonnie 172342.20 +PREHOOK: query: show tblproperties cmv_mat_view +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} +numFiles 1 +totalSize 453 +#### A masked pattern was here #### +PREHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from db1.cmv_basetable where a = 3 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: db1@cmv_basetable +PREHOOK: Output: database:db2 +PREHOOK: Output: db2@cmv_mat_view2 +POSTHOOK: query: create materialized view if not exists cmv_mat_view2 enable rewrite +as select a, c from db1.cmv_basetable where a = 3 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: db1@cmv_basetable +POSTHOOK: Output: database:db2 +POSTHOOK: Output: db2@cmv_mat_view2 +PREHOOK: query: select * from cmv_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: db2@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select * from cmv_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: db2@cmv_mat_view2 +#### A masked pattern was here #### +3 978.76 +3 9.80 +PREHOOK: query: show tblproperties cmv_mat_view2 +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view2 +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} +numFiles 1 +totalSize 322 +#### A masked pattern was here #### +PREHOOK: query: create database db3 +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:db3 +POSTHOOK: query: create database db3 +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:db3 +PREHOOK: query: use db3 +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:db3 +POSTHOOK: query: use db3 +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:db3 +PREHOOK: query: explain +select a, c from db1.cmv_basetable where a = 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a, c from db1.cmv_basetable where a = 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: db2.cmv_mat_view2 + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: select a, c from db1.cmv_basetable where a = 3 +PREHOOK: type: QUERY +PREHOOK: Input: db1@cmv_basetable +PREHOOK: Input: db2@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select a, c from db1.cmv_basetable where a = 3 +POSTHOOK: type: QUERY +POSTHOOK: Input: db1@cmv_basetable +POSTHOOK: Input: db2@cmv_mat_view2 +#### A masked pattern was here #### +3 978.76 +3 9.80 diff --git ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out new file mode 100644 index 0000000000..2be1536453 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out @@ -0,0 +1,354 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values (1, 'alfred', 10.30),(2, 'bob', 3.14),(2, 'bonnie', 172342.2),(3, 'calvin', 978.76),(3, 'charlie', 9.8) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +PREHOOK: query: create materialized view cmv_mat_view +comment 'this is the first view' +tblproperties ('key'='foo') as select a, c from cmv_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: create materialized view cmv_mat_view +comment 'this is the first view' +tblproperties ('key'='foo') as select a, c from cmv_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: describe cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: describe cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +a int +c decimal(10,2) +PREHOOK: query: describe extended cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: describe extended cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +a int +c decimal(10,2) + +#### A masked pattern was here #### +PREHOOK: query: describe formatted cmv_mat_view +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view +POSTHOOK: query: describe formatted cmv_mat_view +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view +# col_name data_type comment +a int +c decimal(10,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {} + comment this is the first view + key foo + numFiles 1 + totalSize 346 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select a, c from cmv_basetable +View Expanded Text: select `cmv_basetable`.`a`, `cmv_basetable`.`c` from `default`.`cmv_basetable` +View Rewrite Enabled: No +PREHOOK: query: show tblproperties cmv_mat_view +PREHOOK: type: SHOW_TBLPROPERTIES +POSTHOOK: query: show tblproperties cmv_mat_view +POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} +comment this is the first view +key foo +numFiles 1 +totalSize 346 +#### A masked pattern was here #### +PREHOOK: query: select a, c from cmv_mat_view +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: select a, c from cmv_mat_view +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 10.30 +2 3.14 +2 172342.20 +3 978.76 +3 9.80 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: create materialized view cmv_mat_view2 +comment 'this is the second view' +stored as textfile +tblproperties ('key'='alice','key2'='bob') as select a from cmv_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: create materialized view cmv_mat_view2 +comment 'this is the second view' +stored as textfile +tblproperties ('key'='alice','key2'='bob') as select a from cmv_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: describe formatted cmv_mat_view2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view2 +POSTHOOK: query: describe formatted cmv_mat_view2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view2 +# col_name data_type comment +a int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {} + comment this is the second view + key alice + key2 bob + numFiles 1 + totalSize 10 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select a from cmv_basetable +View Expanded Text: select `cmv_basetable`.`a` from `default`.`cmv_basetable` +View Rewrite Enabled: No +PREHOOK: query: select a from cmv_mat_view2 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +POSTHOOK: query: select a from cmv_mat_view2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view2 +#### A masked pattern was here #### +1 +2 +2 +3 +3 +PREHOOK: query: drop materialized view cmv_mat_view2 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view2 +PREHOOK: Output: default@cmv_mat_view2 +POSTHOOK: query: drop materialized view cmv_mat_view2 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view2 +POSTHOOK: Output: default@cmv_mat_view2 +PREHOOK: query: create materialized view cmv_mat_view3 +comment 'this is the third view' +row format + delimited fields terminated by '\t' +as select * from cmv_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view3 +POSTHOOK: query: create materialized view cmv_mat_view3 +comment 'this is the third view' +row format + delimited fields terminated by '\t' +as select * from cmv_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view3 +PREHOOK: query: describe formatted cmv_mat_view3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view3 +POSTHOOK: query: describe formatted cmv_mat_view3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view3 +# col_name data_type comment +a int +b varchar(256) +c decimal(10,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {} + comment this is the third view + numFiles 1 + totalSize 497 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select * from cmv_basetable +View Expanded Text: select `cmv_basetable`.`a`, `cmv_basetable`.`b`, `cmv_basetable`.`c` from `default`.`cmv_basetable` +View Rewrite Enabled: No +PREHOOK: query: select a, b, c from cmv_mat_view3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +POSTHOOK: query: select a, b, c from cmv_mat_view3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +1 alfred 10.30 +2 bob 3.14 +2 bonnie 172342.20 +3 calvin 978.76 +3 charlie 9.80 +PREHOOK: query: select distinct a from cmv_mat_view3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +POSTHOOK: query: select distinct a from cmv_mat_view3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view3 +#### A masked pattern was here #### +1 +2 +3 +PREHOOK: query: drop materialized view cmv_mat_view3 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view3 +PREHOOK: Output: default@cmv_mat_view3 +POSTHOOK: query: drop materialized view cmv_mat_view3 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view3 +POSTHOOK: Output: default@cmv_mat_view3 +PREHOOK: query: create materialized view cmv_mat_view4 +comment 'this is the last view' +stored as textfile +#### A masked pattern was here #### +as select a from cmv_basetable +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view4 +POSTHOOK: query: create materialized view cmv_mat_view4 +comment 'this is the last view' +stored as textfile +#### A masked pattern was here #### +as select a from cmv_basetable +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view4 +PREHOOK: query: describe formatted cmv_mat_view4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cmv_mat_view4 +POSTHOOK: query: describe formatted cmv_mat_view4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cmv_mat_view4 +# col_name data_type comment +a int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {} + comment this is the last view + numFiles 1 + totalSize 10 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select a from cmv_basetable +View Expanded Text: select `cmv_basetable`.`a` from `default`.`cmv_basetable` +View Rewrite Enabled: No +PREHOOK: query: select a from cmv_mat_view4 +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_mat_view4 +#### A masked pattern was here #### +POSTHOOK: query: select a from cmv_mat_view4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_mat_view4 +#### A masked pattern was here #### +1 +2 +2 +3 +3 +PREHOOK: query: drop materialized view cmv_mat_view4 +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view4 +PREHOOK: Output: default@cmv_mat_view4 +POSTHOOK: query: drop materialized view cmv_mat_view4 +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view4 +POSTHOOK: Output: default@cmv_mat_view4 diff --git ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out new file mode 100644 index 0000000000..ceb34f7b13 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out @@ -0,0 +1,39 @@ +PREHOOK: query: create materialized view dmv_mat_view as select cint, cstring1 from alltypesorc where cint < 0 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@dmv_mat_view +POSTHOOK: query: create materialized view dmv_mat_view as select cint, cstring1 from alltypesorc where cint < 0 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dmv_mat_view +PREHOOK: query: show table extended like dmv_mat_view +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like dmv_mat_view +POSTHOOK: type: SHOW_TABLESTATUS +tableName:dmv_mat_view +#### A masked pattern was here #### +inputformat:org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +outputformat:org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +columns:struct columns { i32 cint, string cstring1} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:47120 +maxFileSize:47120 +minFileSize:47120 +#### A masked pattern was here #### + +PREHOOK: query: drop materialized view dmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@dmv_mat_view +PREHOOK: Output: default@dmv_mat_view +POSTHOOK: query: drop materialized view dmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@dmv_mat_view +POSTHOOK: Output: default@dmv_mat_view +PREHOOK: query: show table extended like dmv_mat_view +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like dmv_mat_view +POSTHOOK: type: SHOW_TABLESTATUS diff --git ql/src/test/results/clientpositive/llap/materialized_view_rewrite_ssb.q.out ql/src/test/results/clientpositive/llap/materialized_view_rewrite_ssb.q.out new file mode 100644 index 0000000000..269f202220 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_rewrite_ssb.q.out @@ -0,0 +1,1671 @@ +PREHOOK: query: CREATE TABLE `customer`( + `c_custkey` bigint, + `c_name` string, + `c_address` string, + `c_city` string, + `c_nation` string, + `c_region` string, + `c_phone` string, + `c_mktsegment` string, + primary key (`c_custkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@customer +POSTHOOK: query: CREATE TABLE `customer`( + `c_custkey` bigint, + `c_name` string, + `c_address` string, + `c_city` string, + `c_nation` string, + `c_region` string, + `c_phone` string, + `c_mktsegment` string, + primary key (`c_custkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@customer +PREHOOK: query: CREATE TABLE `dates`( + `d_datekey` bigint, + `d_date` string, + `d_dayofweek` string, + `d_month` string, + `d_year` int, + `d_yearmonthnum` int, + `d_yearmonth` string, + `d_daynuminweek` int, + `d_daynuminmonth` int, + `d_daynuminyear` int, + `d_monthnuminyear` int, + `d_weeknuminyear` int, + `d_sellingseason` string, + `d_lastdayinweekfl` int, + `d_lastdayinmonthfl` int, + `d_holidayfl` int , + `d_weekdayfl`int, + primary key (`d_datekey`) disable rely +) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dates +POSTHOOK: query: CREATE TABLE `dates`( + `d_datekey` bigint, + `d_date` string, + `d_dayofweek` string, + `d_month` string, + `d_year` int, + `d_yearmonthnum` int, + `d_yearmonth` string, + `d_daynuminweek` int, + `d_daynuminmonth` int, + `d_daynuminyear` int, + `d_monthnuminyear` int, + `d_weeknuminyear` int, + `d_sellingseason` string, + `d_lastdayinweekfl` int, + `d_lastdayinmonthfl` int, + `d_holidayfl` int , + `d_weekdayfl`int, + primary key (`d_datekey`) disable rely +) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dates +PREHOOK: query: CREATE TABLE `ssb_part`( + `p_partkey` bigint, + `p_name` string, + `p_mfgr` string, + `p_category` string, + `p_brand1` string, + `p_color` string, + `p_type` string, + `p_size` int, + `p_container` string, + primary key (`p_partkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ssb_part +POSTHOOK: query: CREATE TABLE `ssb_part`( + `p_partkey` bigint, + `p_name` string, + `p_mfgr` string, + `p_category` string, + `p_brand1` string, + `p_color` string, + `p_type` string, + `p_size` int, + `p_container` string, + primary key (`p_partkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ssb_part +PREHOOK: query: CREATE TABLE `supplier`( + `s_suppkey` bigint, + `s_name` string, + `s_address` string, + `s_city` string, + `s_nation` string, + `s_region` string, + `s_phone` string, + primary key (`s_suppkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@supplier +POSTHOOK: query: CREATE TABLE `supplier`( + `s_suppkey` bigint, + `s_name` string, + `s_address` string, + `s_city` string, + `s_nation` string, + `s_region` string, + `s_phone` string, + primary key (`s_suppkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@supplier +PREHOOK: query: CREATE TABLE `lineorder`( + `lo_orderkey` bigint, + `lo_linenumber` int, + `lo_custkey` bigint not null disable rely, + `lo_partkey` bigint not null disable rely, + `lo_suppkey` bigint not null disable rely, + `lo_orderdate` bigint not null disable rely, + `lo_ordpriority` string, + `lo_shippriority` string, + `lo_quantity` double, + `lo_extendedprice` double, + `lo_ordtotalprice` double, + `lo_discount` double, + `lo_revenue` double, + `lo_supplycost` double, + `lo_tax` double, + `lo_commitdate` bigint, + `lo_shipmode` string, + primary key (`lo_orderkey`) disable rely, + constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, + constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, + constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, + constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lineorder +POSTHOOK: query: CREATE TABLE `lineorder`( + `lo_orderkey` bigint, + `lo_linenumber` int, + `lo_custkey` bigint not null disable rely, + `lo_partkey` bigint not null disable rely, + `lo_suppkey` bigint not null disable rely, + `lo_orderdate` bigint not null disable rely, + `lo_ordpriority` string, + `lo_shippriority` string, + `lo_quantity` double, + `lo_extendedprice` double, + `lo_ordtotalprice` double, + `lo_discount` double, + `lo_revenue` double, + `lo_supplycost` double, + `lo_tax` double, + `lo_commitdate` bigint, + `lo_shipmode` string, + primary key (`lo_orderkey`) disable rely, + constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, + constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, + constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, + constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lineorder +PREHOOK: query: analyze table customer compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@customer +PREHOOK: Output: default@customer +#### A masked pattern was here #### +POSTHOOK: query: analyze table customer compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer +POSTHOOK: Output: default@customer +#### A masked pattern was here #### +PREHOOK: query: analyze table dates compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dates +PREHOOK: Output: default@dates +#### A masked pattern was here #### +POSTHOOK: query: analyze table dates compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dates +POSTHOOK: Output: default@dates +#### A masked pattern was here #### +PREHOOK: query: analyze table ssb_part compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@ssb_part +PREHOOK: Output: default@ssb_part +#### A masked pattern was here #### +POSTHOOK: query: analyze table ssb_part compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ssb_part +POSTHOOK: Output: default@ssb_part +#### A masked pattern was here #### +PREHOOK: query: analyze table supplier compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@supplier +PREHOOK: Output: default@supplier +#### A masked pattern was here #### +POSTHOOK: query: analyze table supplier compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@supplier +POSTHOOK: Output: default@supplier +#### A masked pattern was here #### +PREHOOK: query: analyze table lineorder compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@lineorder +PREHOOK: Output: default@lineorder +#### A masked pattern was here #### +POSTHOOK: query: analyze table lineorder compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lineorder +POSTHOOK: Output: default@lineorder +#### A masked pattern was here #### +PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE +AS +SELECT + c_city, + c_nation, + c_region, + d_weeknuminyear, + d_year, + d_yearmonth, + d_yearmonthnum, + lo_discount, + lo_quantity, + p_brand1, + p_category, + p_mfgr, + s_city, + s_nation, + s_region, + lo_revenue, + lo_extendedprice * lo_discount discounted_price, + lo_revenue - lo_supplycost net_revenue +FROM + customer, dates, lineorder, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and lo_custkey = c_custkey +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@customer +PREHOOK: Input: default@dates +PREHOOK: Input: default@lineorder +PREHOOK: Input: default@ssb_part +PREHOOK: Input: default@supplier +PREHOOK: Output: database:default +PREHOOK: Output: default@ssb_mv +POSTHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE +AS +SELECT + c_city, + c_nation, + c_region, + d_weeknuminyear, + d_year, + d_yearmonth, + d_yearmonthnum, + lo_discount, + lo_quantity, + p_brand1, + p_category, + p_mfgr, + s_city, + s_nation, + s_region, + lo_revenue, + lo_extendedprice * lo_discount discounted_price, + lo_revenue - lo_supplycost net_revenue +FROM + customer, dates, lineorder, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and lo_custkey = c_custkey +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@customer +POSTHOOK: Input: default@dates +POSTHOOK: Input: default@lineorder +POSTHOOK: Input: default@ssb_part +POSTHOOK: Input: default@supplier +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ssb_mv +PREHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_year = 1993 + and lo_discount between 1 and 3 + and lo_quantity < 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_year = 1993 + and lo_discount between 1 and 3 + and lo_quantity < 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((d_year = 1993) and (lo_quantity < 25.0) and lo_discount BETWEEN 1.0 AND 3.0) (type: boolean) + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: discounted_price (type: double) + outputColumnNames: discounted_price + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(discounted_price) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_yearmonthnum = 199401 + and lo_discount between 4 and 6 + and lo_quantity between 26 and 35 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_yearmonthnum = 199401 + and lo_discount between 4 and 6 + and lo_quantity between 26 and 35 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((d_yearmonthnum = 199401) and lo_discount BETWEEN 4.0 AND 6.0 and lo_quantity BETWEEN 26.0 AND 35.0) (type: boolean) + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: discounted_price (type: double) + outputColumnNames: discounted_price + Statistics: Num rows: 1 Data size: 28 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(discounted_price) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_weeknuminyear = 6 + and d_year = 1994 + and lo_discount between 5 and 7 + and lo_quantity between 26 and 35 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_weeknuminyear = 6 + and d_year = 1994 + and lo_discount between 5 and 7 + and lo_quantity between 26 and 35 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((d_weeknuminyear = 6) and (d_year = 1994) and lo_discount BETWEEN 5.0 AND 7.0 and lo_quantity BETWEEN 26.0 AND 35.0) (type: boolean) + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: discounted_price (type: double) + outputColumnNames: discounted_price + Statistics: Num rows: 1 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(discounted_price) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 40 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_category = 'MFGR#12' + and s_region = 'AMERICA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_category = 'MFGR#12' + and s_region = 'AMERICA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((p_category = 'MFGR#12') and (s_region = 'AMERICA')) (type: boolean) + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: d_year (type: int), p_brand1 (type: string), lo_revenue (type: double) + outputColumnNames: d_year, p_brand1, lo_revenue + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int), p_brand1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int), _col2 (type: string) + sort order: ++ + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 between 'MFGR#2221' and 'MFGR#2228' + and s_region = 'ASIA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 between 'MFGR#2221' and 'MFGR#2228' + and s_region = 'ASIA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s_region = 'ASIA') and p_brand1 BETWEEN 'MFGR#2221' AND 'MFGR#2228') (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: d_year (type: int), p_brand1 (type: string), lo_revenue (type: double) + outputColumnNames: d_year, p_brand1, lo_revenue + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int), p_brand1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int), _col2 (type: string) + sort order: ++ + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 = 'MFGR#2239' + and s_region = 'EUROPE' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 = 'MFGR#2239' + and s_region = 'EUROPE' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((p_brand1 = 'MFGR#2239') and (s_region = 'EUROPE')) (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: d_year (type: int), lo_revenue (type: double) + outputColumnNames: d_year, lo_revenue + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), 'MFGR#2239' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_nation, s_nation, d_year, + sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_region = 'ASIA' + and s_region = 'ASIA' + and d_year >= 1992 and d_year <= 1997 +group by + c_nation, s_nation, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_nation, s_nation, d_year, + sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_region = 'ASIA' + and s_region = 'ASIA' + and d_year >= 1992 and d_year <= 1997 +group by + c_nation, s_nation, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((c_region = 'ASIA') and (d_year <= 1997) and (d_year >= 1992) and (s_region = 'ASIA')) (type: boolean) + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_nation (type: string), d_year (type: int), s_nation (type: string), lo_revenue (type: double) + outputColumnNames: c_nation, d_year, s_nation, lo_revenue + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int), c_nation (type: string), s_nation (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_nation = 'UNITED STATES' + and s_nation = 'UNITED STATES' + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_nation = 'UNITED STATES' + and s_nation = 'UNITED STATES' + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((c_nation = 'UNITED STATES') and (d_year <= 1997) and (d_year >= 1992) and (s_nation = 'UNITED STATES')) (type: boolean) + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_city (type: string), d_year (type: int), s_city (type: string), lo_revenue (type: double) + outputColumnNames: c_city, d_year, s_city, lo_revenue + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int), c_city (type: string), s_city (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (d_year <= 1997) and (d_year >= 1992)) (type: boolean) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int), c_city (type: string), s_city (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 380 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_yearmonth = 'Dec1997' +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_yearmonth = 'Dec1997' +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (d_yearmonth = 'Dec1997')) (type: boolean) + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_city (type: string), d_year (type: int), s_city (type: string), lo_revenue (type: double) + outputColumnNames: c_city, d_year, s_city, lo_revenue + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(lo_revenue) + keys: d_year (type: int), c_city (type: string), s_city (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 564 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + d_year, c_nation, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, c_nation +order by + d_year, c_nation +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + d_year, c_nation, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, c_nation +order by + d_year, c_nation +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_nation (type: string), d_year (type: int), net_revenue (type: double) + outputColumnNames: c_nation, d_year, net_revenue + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(net_revenue) + keys: d_year (type: int), c_nation (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 748 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + d_year, s_nation, p_category, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (d_year = 1997 or d_year = 1998) + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, s_nation, p_category +order by + d_year, s_nation, p_category +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + d_year, s_nation, p_category, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (d_year = 1997 or d_year = 1998) + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, s_nation, p_category +order by + d_year, s_nation, p_category +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((d_year = 1997) or (d_year = 1998)) and ((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: d_year (type: int), p_category (type: string), s_nation (type: string), net_revenue (type: double) + outputColumnNames: d_year, p_category, s_nation, net_revenue + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(net_revenue) + keys: d_year (type: int), s_nation (type: string), p_category (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + d_year, s_city, p_brand1, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_nation = 'UNITED STATES' + and (d_year = 1997 or d_year = 1998) + and p_category = 'MFGR#14' +group by + d_year, s_city, p_brand1 +order by + d_year, s_city, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + d_year, s_city, p_brand1, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_nation = 'UNITED STATES' + and (d_year = 1997 or d_year = 1998) + and p_category = 'MFGR#14' +group by + d_year, s_city, p_brand1 +order by + d_year, s_city, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((d_year = 1997) or (d_year = 1998)) and (c_region = 'AMERICA') and (p_category = 'MFGR#14') and (s_nation = 'UNITED STATES')) (type: boolean) + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: d_year (type: int), p_brand1 (type: string), s_city (type: string), net_revenue (type: double) + outputColumnNames: d_year, p_brand1, s_city, net_revenue + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(net_revenue) + keys: d_year (type: int), s_city (type: string), p_brand1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 932 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@ssb_mv +PREHOOK: Output: default@ssb_mv +POSTHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@ssb_mv +POSTHOOK: Output: default@ssb_mv diff --git ql/src/test/results/clientpositive/llap/materialized_view_rewrite_ssb_2.q.out ql/src/test/results/clientpositive/llap/materialized_view_rewrite_ssb_2.q.out new file mode 100644 index 0000000000..cae4187bf5 --- /dev/null +++ ql/src/test/results/clientpositive/llap/materialized_view_rewrite_ssb_2.q.out @@ -0,0 +1,1677 @@ +PREHOOK: query: CREATE TABLE `customer`( + `c_custkey` bigint, + `c_name` string, + `c_address` string, + `c_city` string, + `c_nation` string, + `c_region` string, + `c_phone` string, + `c_mktsegment` string, + primary key (`c_custkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@customer +POSTHOOK: query: CREATE TABLE `customer`( + `c_custkey` bigint, + `c_name` string, + `c_address` string, + `c_city` string, + `c_nation` string, + `c_region` string, + `c_phone` string, + `c_mktsegment` string, + primary key (`c_custkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@customer +PREHOOK: query: CREATE TABLE `dates`( + `d_datekey` bigint, + `d_date` string, + `d_dayofweek` string, + `d_month` string, + `d_year` int, + `d_yearmonthnum` int, + `d_yearmonth` string, + `d_daynuminweek` int, + `d_daynuminmonth` int, + `d_daynuminyear` int, + `d_monthnuminyear` int, + `d_weeknuminyear` int, + `d_sellingseason` string, + `d_lastdayinweekfl` int, + `d_lastdayinmonthfl` int, + `d_holidayfl` int , + `d_weekdayfl`int, + primary key (`d_datekey`) disable rely +) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dates +POSTHOOK: query: CREATE TABLE `dates`( + `d_datekey` bigint, + `d_date` string, + `d_dayofweek` string, + `d_month` string, + `d_year` int, + `d_yearmonthnum` int, + `d_yearmonth` string, + `d_daynuminweek` int, + `d_daynuminmonth` int, + `d_daynuminyear` int, + `d_monthnuminyear` int, + `d_weeknuminyear` int, + `d_sellingseason` string, + `d_lastdayinweekfl` int, + `d_lastdayinmonthfl` int, + `d_holidayfl` int , + `d_weekdayfl`int, + primary key (`d_datekey`) disable rely +) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dates +PREHOOK: query: CREATE TABLE `ssb_part`( + `p_partkey` bigint, + `p_name` string, + `p_mfgr` string, + `p_category` string, + `p_brand1` string, + `p_color` string, + `p_type` string, + `p_size` int, + `p_container` string, + primary key (`p_partkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ssb_part +POSTHOOK: query: CREATE TABLE `ssb_part`( + `p_partkey` bigint, + `p_name` string, + `p_mfgr` string, + `p_category` string, + `p_brand1` string, + `p_color` string, + `p_type` string, + `p_size` int, + `p_container` string, + primary key (`p_partkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ssb_part +PREHOOK: query: CREATE TABLE `supplier`( + `s_suppkey` bigint, + `s_name` string, + `s_address` string, + `s_city` string, + `s_nation` string, + `s_region` string, + `s_phone` string, + primary key (`s_suppkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@supplier +POSTHOOK: query: CREATE TABLE `supplier`( + `s_suppkey` bigint, + `s_name` string, + `s_address` string, + `s_city` string, + `s_nation` string, + `s_region` string, + `s_phone` string, + primary key (`s_suppkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@supplier +PREHOOK: query: CREATE TABLE `lineorder`( + `lo_orderkey` bigint, + `lo_linenumber` int, + `lo_custkey` bigint not null disable rely, + `lo_partkey` bigint not null disable rely, + `lo_suppkey` bigint not null disable rely, + `lo_orderdate` bigint not null disable rely, + `lo_ordpriority` string, + `lo_shippriority` string, + `lo_quantity` double, + `lo_extendedprice` double, + `lo_ordtotalprice` double, + `lo_discount` double, + `lo_revenue` double, + `lo_supplycost` double, + `lo_tax` double, + `lo_commitdate` bigint, + `lo_shipmode` string, + primary key (`lo_orderkey`) disable rely, + constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, + constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, + constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, + constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@lineorder +POSTHOOK: query: CREATE TABLE `lineorder`( + `lo_orderkey` bigint, + `lo_linenumber` int, + `lo_custkey` bigint not null disable rely, + `lo_partkey` bigint not null disable rely, + `lo_suppkey` bigint not null disable rely, + `lo_orderdate` bigint not null disable rely, + `lo_ordpriority` string, + `lo_shippriority` string, + `lo_quantity` double, + `lo_extendedprice` double, + `lo_ordtotalprice` double, + `lo_discount` double, + `lo_revenue` double, + `lo_supplycost` double, + `lo_tax` double, + `lo_commitdate` bigint, + `lo_shipmode` string, + primary key (`lo_orderkey`) disable rely, + constraint fk1 foreign key (`lo_custkey`) references `customer`(`c_custkey`) disable rely, + constraint fk2 foreign key (`lo_orderdate`) references `dates`(`d_datekey`) disable rely, + constraint fk3 foreign key (`lo_partkey`) references `ssb_part`(`p_partkey`) disable rely, + constraint fk4 foreign key (`lo_suppkey`) references `supplier`(`s_suppkey`) disable rely) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@lineorder +PREHOOK: query: analyze table customer compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@customer +PREHOOK: Output: default@customer +#### A masked pattern was here #### +POSTHOOK: query: analyze table customer compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@customer +POSTHOOK: Output: default@customer +#### A masked pattern was here #### +PREHOOK: query: analyze table dates compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@dates +PREHOOK: Output: default@dates +#### A masked pattern was here #### +POSTHOOK: query: analyze table dates compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dates +POSTHOOK: Output: default@dates +#### A masked pattern was here #### +PREHOOK: query: analyze table ssb_part compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@ssb_part +PREHOOK: Output: default@ssb_part +#### A masked pattern was here #### +POSTHOOK: query: analyze table ssb_part compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ssb_part +POSTHOOK: Output: default@ssb_part +#### A masked pattern was here #### +PREHOOK: query: analyze table supplier compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@supplier +PREHOOK: Output: default@supplier +#### A masked pattern was here #### +POSTHOOK: query: analyze table supplier compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@supplier +POSTHOOK: Output: default@supplier +#### A masked pattern was here #### +PREHOOK: query: analyze table lineorder compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@lineorder +PREHOOK: Output: default@lineorder +#### A masked pattern was here #### +POSTHOOK: query: analyze table lineorder compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@lineorder +POSTHOOK: Output: default@lineorder +#### A masked pattern was here #### +PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE +AS +SELECT + cast(d_year || '-' || d_monthnuminyear || '-' || d_daynuminmonth as timestamp) as `__time`, + c_city, + c_nation, + c_region, + cast(d_weeknuminyear as string) d_weeknuminyear, + cast(d_year as string) d_year, + d_yearmonth, + cast(d_yearmonthnum as string) d_yearmonthnum, + cast(lo_discount as string) lo_discount, + cast(lo_quantity as string) lo_quantity, + p_brand1, + p_category, + p_mfgr, + s_city, + s_nation, + s_region, + lo_revenue, + lo_extendedprice * lo_discount discounted_price, + lo_revenue - lo_supplycost net_revenue +FROM + customer, dates, lineorder, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and lo_custkey = c_custkey +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@customer +PREHOOK: Input: default@dates +PREHOOK: Input: default@lineorder +PREHOOK: Input: default@ssb_part +PREHOOK: Input: default@supplier +PREHOOK: Output: database:default +PREHOOK: Output: default@ssb_mv +POSTHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE +AS +SELECT + cast(d_year || '-' || d_monthnuminyear || '-' || d_daynuminmonth as timestamp) as `__time`, + c_city, + c_nation, + c_region, + cast(d_weeknuminyear as string) d_weeknuminyear, + cast(d_year as string) d_year, + d_yearmonth, + cast(d_yearmonthnum as string) d_yearmonthnum, + cast(lo_discount as string) lo_discount, + cast(lo_quantity as string) lo_quantity, + p_brand1, + p_category, + p_mfgr, + s_city, + s_nation, + s_region, + lo_revenue, + lo_extendedprice * lo_discount discounted_price, + lo_revenue - lo_supplycost net_revenue +FROM + customer, dates, lineorder, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and lo_custkey = c_custkey +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@customer +POSTHOOK: Input: default@dates +POSTHOOK: Input: default@lineorder +POSTHOOK: Input: default@ssb_part +POSTHOOK: Input: default@supplier +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ssb_mv +PREHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_year = 1993 + and lo_discount between 1 and 3 + and lo_quantity < 25 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_year = 1993 + and lo_discount between 1 and 3 + and lo_quantity < 25 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(lo_quantity) < 25.0) and (UDFToInteger(d_year) = 1993) and UDFToDouble(lo_discount) BETWEEN 1.0 AND 3.0) (type: boolean) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: discounted_price (type: double) + outputColumnNames: discounted_price + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(discounted_price) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_yearmonthnum = 199401 + and lo_discount between 4 and 6 + and lo_quantity between 26 and 35 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_yearmonthnum = 199401 + and lo_discount between 4 and 6 + and lo_quantity between 26 and 35 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToInteger(d_yearmonthnum) = 199401) and UDFToDouble(lo_discount) BETWEEN 4.0 AND 6.0 and UDFToDouble(lo_quantity) BETWEEN 26.0 AND 35.0) (type: boolean) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: discounted_price (type: double) + outputColumnNames: discounted_price + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(discounted_price) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 568 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_weeknuminyear = 6 + and d_year = 1994 + and lo_discount between 5 and 7 + and lo_quantity between 26 and 35 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_extendedprice*lo_discount) as revenue +from + lineorder, dates +where + lo_orderdate = d_datekey + and d_weeknuminyear = 6 + and d_year = 1994 + and lo_discount between 5 and 7 + and lo_quantity between 26 and 35 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToInteger(d_weeknuminyear) = 6) and (UDFToInteger(d_year) = 1994) and UDFToDouble(lo_discount) BETWEEN 5.0 AND 7.0 and UDFToDouble(lo_quantity) BETWEEN 26.0 AND 35.0) (type: boolean) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: discounted_price (type: double) + outputColumnNames: discounted_price + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(discounted_price) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 752 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_category = 'MFGR#12' + and s_region = 'AMERICA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_category = 'MFGR#12' + and s_region = 'AMERICA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((p_category = 'MFGR#12') and (s_region = 'AMERICA')) (type: boolean) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(d_year) (type: int), p_brand1 (type: string), lo_revenue (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col2) + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int), _col2 (type: string) + sort order: ++ + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 between 'MFGR#2221' and 'MFGR#2228' + and s_region = 'ASIA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 between 'MFGR#2221' and 'MFGR#2228' + and s_region = 'ASIA' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s_region = 'ASIA') and p_brand1 BETWEEN 'MFGR#2221' AND 'MFGR#2228') (type: boolean) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(d_year) (type: int), p_brand1 (type: string), lo_revenue (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col2) + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: double), _col0 (type: int), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int), _col2 (type: string) + sort order: ++ + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 = 'MFGR#2239' + and s_region = 'EUROPE' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + sum(lo_revenue) as lo_revenue, d_year, p_brand1 +from + lineorder, dates, ssb_part, supplier +where + lo_orderdate = d_datekey + and lo_partkey = p_partkey + and lo_suppkey = s_suppkey + and p_brand1 = 'MFGR#2239' + and s_region = 'EUROPE' +group by + d_year, p_brand1 +order by + d_year, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((p_brand1 = 'MFGR#2239') and (s_region = 'EUROPE')) (type: boolean) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(d_year) (type: int), lo_revenue (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col1) + keys: _col0 (type: int) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: double), _col0 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), KEY.reducesinkkey0 (type: int), 'MFGR#2239' (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_nation, s_nation, d_year, + sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_region = 'ASIA' + and s_region = 'ASIA' + and d_year >= 1992 and d_year <= 1997 +group by + c_nation, s_nation, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_nation, s_nation, d_year, + sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_region = 'ASIA' + and s_region = 'ASIA' + and d_year >= 1992 and d_year <= 1997 +group by + c_nation, s_nation, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToInteger(d_year) <= 1997) and (UDFToInteger(d_year) >= 1992) and (c_region = 'ASIA') and (s_region = 'ASIA')) (type: boolean) + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_nation (type: string), s_nation (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col3) + keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_nation = 'UNITED STATES' + and s_nation = 'UNITED STATES' + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and c_nation = 'UNITED STATES' + and s_nation = 'UNITED STATES' + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToInteger(d_year) <= 1997) and (UDFToInteger(d_year) >= 1992) and (c_nation = 'UNITED STATES') and (s_nation = 'UNITED STATES')) (type: boolean) + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_city (type: string), s_city (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col3) + keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_year >= 1992 and d_year <= 1997 +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (UDFToInteger(d_year) <= 1997) and (UDFToInteger(d_year) >= 1992)) (type: boolean) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_city (type: string), s_city (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col3) + keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_yearmonth = 'Dec1997' +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + c_city, s_city, d_year, sum(lo_revenue) as lo_revenue +from + customer, lineorder, supplier, dates +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_orderdate = d_datekey + and (c_city='UNITED KI1' or c_city='UNITED KI5') + and (s_city='UNITED KI1' or s_city='UNITED KI5') + and d_yearmonth = 'Dec1997' +group by + c_city, s_city, d_year +order by + d_year asc, lo_revenue desc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((c_city = 'UNITED KI1') or (c_city = 'UNITED KI5')) and ((s_city = 'UNITED KI1') or (s_city = 'UNITED KI5')) and (d_yearmonth = 'Dec1997')) (type: boolean) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c_city (type: string), s_city (type: string), UDFToInteger(d_year) (type: int), lo_revenue (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col3) + keys: _col2 (type: int), _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col0 (type: int), _col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: double) + sort order: +- + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 744 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + d_year, c_nation, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, c_nation +order by + d_year, c_nation +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + d_year, c_nation, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, c_nation +order by + d_year, c_nation +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(d_year) (type: int), c_nation (type: string), net_revenue (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col2) + keys: _col0 (type: int), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 928 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + d_year, s_nation, p_category, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (d_year = 1997 or d_year = 1998) + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, s_nation, p_category +order by + d_year, s_nation, p_category +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + d_year, s_nation, p_category, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_region = 'AMERICA' + and (d_year = 1997 or d_year = 1998) + and (p_mfgr = 'MFGR#1' or p_mfgr = 'MFGR#2') +group by + d_year, s_nation, p_category +order by + d_year, s_nation, p_category +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(d_year) = 1997) or (UDFToInteger(d_year) = 1998)) and ((p_mfgr = 'MFGR#1') or (p_mfgr = 'MFGR#2')) and (c_region = 'AMERICA') and (s_region = 'AMERICA')) (type: boolean) + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(d_year) (type: int), s_nation (type: string), p_category (type: string), net_revenue (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col1 (type: string), _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select + d_year, s_city, p_brand1, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_nation = 'UNITED STATES' + and (d_year = 1997 or d_year = 1998) + and p_category = 'MFGR#14' +group by + d_year, s_city, p_brand1 +order by + d_year, s_city, p_brand1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +select + d_year, s_city, p_brand1, + sum(lo_revenue - lo_supplycost) as profit +from + dates, customer, supplier, ssb_part, lineorder +where + lo_custkey = c_custkey + and lo_suppkey = s_suppkey + and lo_partkey = p_partkey + and lo_orderdate = d_datekey + and c_region = 'AMERICA' + and s_nation = 'UNITED STATES' + and (d_year = 1997 or d_year = 1998) + and p_category = 'MFGR#14' +group by + d_year, s_city, p_brand1 +order by + d_year, s_city, p_brand1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: default.ssb_mv + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToInteger(d_year) = 1997) or (UDFToInteger(d_year) = 1998)) and (c_region = 'AMERICA') and (p_category = 'MFGR#14') and (s_nation = 'UNITED STATES')) (type: boolean) + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(d_year) (type: int), s_city (type: string), p_brand1 (type: string), net_revenue (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: sum(_col3) + keys: _col0 (type: int), _col1 (type: string), _col2 (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string) + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + sort order: +++ + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1112 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@ssb_mv +PREHOOK: Output: default@ssb_mv +POSTHOOK: query: DROP MATERIALIZED VIEW `ssb_mv` +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@ssb_mv +POSTHOOK: Output: default@ssb_mv diff --git ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out index befb6e7992..a9d090d5bd 100644 --- ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out +++ ql/src/test/results/clientpositive/llap/selectDistinctStar.q.out @@ -1349,12 +1349,10 @@ STAGE PLANS: Stage: Stage-1 Create View Operator: Create View - or replace: false columns: key string, value string expanded text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 name: default.sdi original text: select distinct * from src order by key limit 2 - rewrite enabled: false PREHOOK: query: create view sdi as select distinct * from src order by key limit 2 PREHOOK: type: CREATEVIEW @@ -3812,12 +3810,10 @@ STAGE PLANS: Stage: Stage-1 Create View Operator: Create View - or replace: false columns: key string, value string expanded text: select distinct `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 2 name: default.sdi original text: select distinct * from src order by key limit 2 - rewrite enabled: false PREHOOK: query: create view sdi as select distinct * from src order by key limit 2 PREHOOK: type: CREATEVIEW diff --git ql/src/test/results/clientpositive/llap/union_top_level.q.out ql/src/test/results/clientpositive/llap/union_top_level.q.out index a010232171..9635cad62f 100644 --- ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -1081,7 +1081,6 @@ STAGE PLANS: Stage: Stage-1 Create View Operator: Create View - or replace: false columns: key string, value int expanded text: select `a`.`key`, `a`.`value` from (select `src`.`key`, 0 as `value` from `default`.`src` where `src`.`key` % 3 == 0 limit 3)`a` union all @@ -1094,7 +1093,6 @@ union all select * from (select key, 1 as value from src where key % 3 == 1 limit 3)b union all select * from (select key, 2 as value from src where key % 3 == 2 limit 3)c - rewrite enabled: false PREHOOK: query: create view union_top_view as select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a diff --git ql/src/test/results/clientpositive/llap/vector_windowing.q.out ql/src/test/results/clientpositive/llap/vector_windowing.q.out index 776ae3e085..a2a705b1a1 100644 --- ql/src/test/results/clientpositive/llap/vector_windowing.q.out +++ ql/src/test/results/clientpositive/llap/vector_windowing.q.out @@ -4418,7 +4418,6 @@ STAGE PLANS: Create View Operator: Create View if not exists: true - or replace: false columns: p_mfgr string, p_brand string, s double expanded text: select `part`.`p_mfgr`, `part`.`p_brand`, round(sum(`part`.`p_retailprice`),2) as `s` @@ -4429,7 +4428,6 @@ group by `part`.`p_mfgr`, `part`.`p_brand` round(sum(p_retailprice),2) as s from part group by p_mfgr, p_brand - rewrite enabled: false PREHOOK: query: create view IF NOT EXISTS mfgr_price_view as select p_mfgr, p_brand, @@ -4762,7 +4760,6 @@ STAGE PLANS: Create View Operator: Create View if not exists: true - or replace: false columns: p_mfgr string, p_brand string, s double expanded text: select `part`.`p_mfgr`, `part`.`p_brand`, round(sum(`part`.`p_retailprice`) over w1,2) as `s` @@ -4773,7 +4770,6 @@ window w1 as (distribute by `part`.`p_mfgr` sort by `part`.`p_name` rows between round(sum(p_retailprice) over w1,2) as s from part window w1 as (distribute by p_mfgr sort by p_name rows between 2 preceding and current row) - rewrite enabled: false PREHOOK: query: create view IF NOT EXISTS mfgr_brand_price_view as select p_mfgr, p_brand, diff --git ql/src/test/results/clientpositive/materialized_view_create.q.out ql/src/test/results/clientpositive/materialized_view_create.q.out index 0fca69f4fa..928618390d 100644 --- ql/src/test/results/clientpositive/materialized_view_create.q.out +++ ql/src/test/results/clientpositive/materialized_view_create.q.out @@ -160,6 +160,7 @@ PREHOOK: query: show tblproperties cmv_mat_view5 PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view5 POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} key value numFiles 1 totalSize 710 diff --git ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out index 81a7950773..aa3240cad4 100644 --- ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out +++ ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out @@ -52,6 +52,7 @@ PREHOOK: query: show tblproperties cmv_mat_view PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} numFiles 1 totalSize 453 #### A masked pattern was here #### @@ -75,12 +76,13 @@ POSTHOOK: query: select * from cmv_mat_view2 POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_mat_view2 #### A masked pattern was here #### -3 978.76 3 9.80 +3 978.76 PREHOOK: query: show tblproperties cmv_mat_view2 PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} numFiles 1 totalSize 322 #### A masked pattern was here #### @@ -117,8 +119,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_mat_view2 #### A masked pattern was here #### -3 978.76 3 9.80 +3 978.76 PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE PREHOOK: Input: default@cmv_mat_view2 diff --git ql/src/test/results/clientpositive/materialized_view_create_rewrite_3.q.out ql/src/test/results/clientpositive/materialized_view_create_rewrite_3.q.out new file mode 100644 index 0000000000..9fd70b6937 --- /dev/null +++ ql/src/test/results/clientpositive/materialized_view_create_rewrite_3.q.out @@ -0,0 +1,531 @@ +PREHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: create table cmv_basetable (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable +PREHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable +POSTHOOK: query: insert into cmv_basetable values + (1, 'alfred', 10.30, 2), + (2, 'bob', 3.14, 3), + (2, 'bonnie', 172342.2, 3), + (3, 'calvin', 978.76, 3), + (3, 'charlie', 9.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable +POSTHOOK: Lineage: cmv_basetable.a EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.b EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.c EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable.d EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_basetable_2 +PREHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (1, 'alfred', 10.30, 2), + (3, 'calvin', 978.76, 3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.b EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.c EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.d EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-5 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 2 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(10,2)) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 5 Data size: 89 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 89 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 5 Data size: 89 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-5 + Create View Operator: + Create View + columns: a int, c decimal(10,2) + expanded text: SELECT `cmv_basetable`.`a`, `cmv_basetable_2`.`c` + FROM `default`.`cmv_basetable` JOIN `default`.`cmv_basetable_2` ON (`cmv_basetable`.`a` = `cmv_basetable_2`.`a`) + WHERE `cmv_basetable_2`.`c` > 10.0 + GROUP BY `cmv_basetable`.`a`, `cmv_basetable_2`.`c` + name: default.cmv_mat_view + original text: SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c + rewrite enabled: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: CREATE MATERIALIZED VIEW cmv_mat_view ENABLE REWRITE AS + SELECT cmv_basetable.a, cmv_basetable_2.c + FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) + WHERE cmv_basetable_2.c > 10.0 + GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 325 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (c > 10.1) (type: boolean) + Statistics: Num rows: 1 Data size: 162 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 162 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 162 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 +3 +PREHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +PREHOOK: type: QUERY +PREHOOK: Output: default@cmv_basetable_2 +POSTHOOK: query: insert into cmv_basetable_2 values + (3, 'charlie', 15.8, 1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@cmv_basetable_2 +POSTHOOK: Lineage: cmv_basetable_2.a EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.b EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.c EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: cmv_basetable_2.d EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 2 Data size: 325 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (c > 10.1) (type: boolean) + Statistics: Num rows: 1 Data size: 162 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 162 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 162 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 +3 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-5 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: cmv_basetable + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: a is not null (type: boolean) + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 5 Data size: 81 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: cmv_basetable_2 + Statistics: Num rows: 3 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((c > 10) and a is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int), c (type: decimal(10,2)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: decimal(10,2)) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 5 Data size: 89 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: int), _col2 (type: decimal(10,2)) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 5 Data size: 89 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: int), _col1 (type: decimal(10,2)) + sort order: ++ + Map-reduce partition columns: _col0 (type: int), _col1 (type: decimal(10,2)) + Statistics: Num rows: 5 Data size: 89 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: int), KEY._col1 (type: decimal(10,2)) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cmv_mat_view + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-5 + Create View Operator: + Create View + columns: a int, c decimal(10,2) + name: default.cmv_mat_view + replace: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cmv_mat_view +PREHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT cmv_basetable.a +FROM cmv_basetable join cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: default.cmv_mat_view + Statistics: Num rows: 3 Data size: 348 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (c > 10.1) (type: boolean) + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +PREHOOK: type: QUERY +PREHOOK: Input: default@cmv_basetable +PREHOOK: Input: default@cmv_basetable_2 +PREHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT cmv_basetable.a +FROM cmv_basetable JOIN cmv_basetable_2 ON (cmv_basetable.a = cmv_basetable_2.a) +WHERE cmv_basetable_2.c > 10.10 +GROUP BY cmv_basetable.a, cmv_basetable_2.c +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cmv_basetable +POSTHOOK: Input: default@cmv_basetable_2 +POSTHOOK: Input: default@cmv_mat_view +#### A masked pattern was here #### +1 +3 +3 +PREHOOK: query: drop materialized view cmv_mat_view +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@cmv_mat_view +PREHOOK: Output: default@cmv_mat_view +POSTHOOK: query: drop materialized view cmv_mat_view +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@cmv_mat_view +POSTHOOK: Output: default@cmv_mat_view diff --git ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out index f715803438..a6d00db0f7 100644 --- ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out +++ ql/src/test/results/clientpositive/materialized_view_create_rewrite_multi_db.q.out @@ -76,6 +76,7 @@ PREHOOK: query: show tblproperties cmv_mat_view PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} numFiles 1 totalSize 453 #### A masked pattern was here #### @@ -105,6 +106,7 @@ PREHOOK: query: show tblproperties cmv_mat_view2 PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} numFiles 1 totalSize 322 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/materialized_view_describe.q.out ql/src/test/results/clientpositive/materialized_view_describe.q.out index 8b7691279c..2be1536453 100644 --- ql/src/test/results/clientpositive/materialized_view_describe.q.out +++ ql/src/test/results/clientpositive/materialized_view_describe.q.out @@ -64,6 +64,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MATERIALIZED_VIEW Table Parameters: + COLUMN_STATS_ACCURATE {} comment this is the first view key foo numFiles 1 @@ -87,6 +88,7 @@ PREHOOK: query: show tblproperties cmv_mat_view PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES +COLUMN_STATS_ACCURATE {} comment this is the first view key foo numFiles 1 @@ -145,6 +147,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MATERIALIZED_VIEW Table Parameters: + COLUMN_STATS_ACCURATE {} comment this is the second view key alice key2 bob @@ -222,6 +225,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MATERIALIZED_VIEW Table Parameters: + COLUMN_STATS_ACCURATE {} comment this is the third view numFiles 1 totalSize 497 @@ -308,6 +312,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MATERIALIZED_VIEW Table Parameters: + COLUMN_STATS_ACCURATE {} comment this is the last view numFiles 1 totalSize 10