diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index a2e6f49e56..7a6ee95015 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -69,4 +69,6 @@ public static final String HADOOP_CREDENTIAL_PROVIDER_PATH_CONFIG = "hadoop.security.credential.provider.path"; public static final String MATERIALIZED_VIEW_REWRITING_TIME_WINDOW = "rewriting.time.window"; + public static final String MATERIALIZED_VIEW_SORT_COLUMNS = "materializedview.sort.columns"; + public static final String MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS = "materializedview.distribute.columns"; } diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 7e3294f554..cb29167d99 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -586,6 +586,7 @@ minillaplocal.query.files=\ load_data_using_job.q,\ load_dyn_part5.q,\ lvj_mapjoin.q,\ + materialized_view_cluster.q,\ materialized_view_create.q,\ materialized_view_create_rewrite_dummy.q,\ materialized_view_create_rewrite_multi_db.q,\ @@ -598,7 +599,9 @@ minillaplocal.query.files=\ materialized_view_create_rewrite_4.q,\ materialized_view_create_rewrite_5.q,\ materialized_view_describe.q,\ + materialized_view_distribute.q,\ materialized_view_drop.q,\ + materialized_view_partition_cluster.q,\ materialized_view_partitioned.q,\ materialized_view_partitioned_2.q,\ materialized_view_partitioned_3.q,\ @@ -620,6 +623,7 @@ minillaplocal.query.files=\ materialized_view_rewrite_part_2.q,\ materialized_view_rewrite_ssb.q,\ materialized_view_rewrite_ssb_2.q,\ + materialized_view_sort.q,\ mapjoin_decimal.q,\ mapjoin_hint.q,\ mapjoin_emit_interval.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/CreateViewDesc.java index 8e7f99bd93..e51465ecd8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/CreateViewDesc.java @@ -23,8 +23,10 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -70,6 +72,10 @@ private String storageHandler; // only used for materialized views private Map serdeProps; // only used for materialized views private Set tablesUsed; // only used for materialized views + private List sortColNames; // only used for materialized views + private List sortCols; // only used for materialized views + private List distributeColNames; // only used for materialized views + private List distributeCols; // only used for materialized views private ReplicationSpec replicationSpec = null; private String ownerName = null; @@ -77,14 +83,17 @@ * Used to create a materialized view descriptor. */ public CreateViewDesc(String viewName, List schema, String comment, Map tblProps, - List partColNames, boolean ifNotExists, boolean orReplace, boolean rewriteEnabled, boolean isAlterViewAs, - String inputFormat, String outputFormat, String location, - String serde, String storageHandler, Map serdeProps) { + List partColNames, List sortColNames, List distributeColNames, + boolean ifNotExists, boolean orReplace, boolean rewriteEnabled, boolean isAlterViewAs, + String inputFormat, String outputFormat, String location, + String serde, String storageHandler, Map serdeProps) { this.viewName = viewName; this.schema = schema; this.comment = comment; this.tblProps = tblProps; this.partColNames = partColNames; + this.sortColNames = sortColNames; + this.distributeColNames = distributeColNames; this.ifNotExists = ifNotExists; this.orReplace = orReplace; @@ -104,8 +113,7 @@ public CreateViewDesc(String viewName, List schema, String comment, */ public CreateViewDesc(String viewName, List schema, String comment, Map tblProps, List partColNames, boolean ifNotExists, boolean orReplace, - boolean isAlterViewAs, - String inputFormat, String outputFormat, String serde) { + boolean isAlterViewAs, String inputFormat, String outputFormat, String serde) { this.viewName = viewName; this.schema = schema; this.comment = comment; @@ -192,6 +200,53 @@ public void setPartColNames(List partColNames) { this.partColNames = partColNames; } + public boolean isOrganized() { + return (sortColNames != null && !sortColNames.isEmpty()) || + (distributeColNames != null && !distributeColNames.isEmpty()); + } + + @Explain(displayName = "sort columns") + public List getSortColsString() { + return Utilities.getFieldSchemaString(sortCols); + } + + public List getSortCols() { + return sortCols; + } + + public void setSortCols(List sortCols) { + this.sortCols = sortCols; + } + + public List getSortColNames() { + return sortColNames; + } + + public void setSortColNames(List sortColNames) { + this.sortColNames = sortColNames; + } + + @Explain(displayName = "distribute columns") + public List getDistributeColsString() { + return Utilities.getFieldSchemaString(distributeCols); + } + + public List getDistributeCols() { + return distributeCols; + } + + public void setDistributeCols(List distributeCols) { + this.distributeCols = distributeCols; + } + + public List getDistributeColNames() { + return distributeColNames; + } + + public void setDistributeColNames(List distributeColNames) { + this.distributeColNames = distributeColNames; + } + @Explain(displayName = "comment") public String getComment() { return comment; @@ -322,12 +377,22 @@ public Table toTable(HiveConf conf) throws HiveException { if (getComment() != null) { tbl.setProperty("comment", getComment()); } - if (getTblProps() != null) { - tbl.getTTable().getParameters().putAll(getTblProps()); + + if (tblProps != null) { + tbl.getParameters().putAll(tblProps); + } + + if (!CollectionUtils.isEmpty(partCols)) { + tbl.setPartCols(partCols); } - if (getPartCols() != null) { - tbl.setPartCols(getPartCols()); + if (!CollectionUtils.isEmpty(sortColNames)) { + tbl.setProperty(Constants.MATERIALIZED_VIEW_SORT_COLUMNS, + Utilities.encodeColumnNames(sortColNames)); + } + if (!CollectionUtils.isEmpty(distributeColNames)) { + tbl.setProperty(Constants.MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS, + Utilities.encodeColumnNames(distributeColNames)); } if (getInputFormat() != null) { @@ -400,4 +465,5 @@ public void setOwnerName(String ownerName) { public String getOwnerName() { return this.ownerName; } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index bc75ec0b56..72d0bd62ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec; +import com.fasterxml.jackson.databind.ObjectMapper; import java.beans.DefaultPersistenceDelegate; import java.beans.Encoder; import java.beans.Expression; @@ -218,6 +219,12 @@ */ @SuppressWarnings({ "nls", "deprecation" }) public final class Utilities { + + /** + * Mapper to use to serialize/deserialize JSON objects (). + */ + public static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + /** * A logger mostly used to trace-log the details of Hive table file operations. Filtering the * logs for FileOperations (with trace logs present) allows one to debug what Hive has done with @@ -4587,4 +4594,20 @@ private static void tryToLogClassPath(String prefix, ClassLoader loader, Logger loader == null ? "null" : loader.getClass().getSimpleName()); } } + + public static String encodeColumnNames(List colNames) throws SemanticException { + try { + return JSON_MAPPER.writeValueAsString(colNames); + } catch (IOException e) { + throw new SemanticException(e); + } + } + + public static List decodeColumnNames(String colNamesStr) throws SemanticException { + try { + return JSON_MAPPER.readValue(colNamesStr, List.class); + } catch (IOException e) { + throw new SemanticException(e); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 9708c47ea6..318e2300ec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.Stack; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; @@ -156,6 +157,14 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } + if (destTable.isMaterializedView() && + (destTable.getProperty(Constants.MATERIALIZED_VIEW_SORT_COLUMNS) != null || + destTable.getProperty(Constants.MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS) != null)) { + LOG.debug("Bailing out of sort dynamic partition optimization as destination is a materialized view" + + "with CLUSTER/SORT/DISTRIBUTE spec"); + return null; + } + // unlink connection between FS and its parent Operator fsParent = fsOp.getParentOperators().get(0); // if all dp columns got constant folded then disable this optimization diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g index 3ca45fe1fe..af9fba6f36 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g @@ -148,6 +148,7 @@ KW_UNIONTYPE: 'UNIONTYPE'; KW_REDUCE: 'REDUCE'; KW_PARTITIONED: 'PARTITIONED'; KW_CLUSTERED: 'CLUSTERED'; +KW_DISTRIBUTED: 'DISTRIBUTED'; KW_SORTED: 'SORTED'; KW_INTO: 'INTO'; KW_BUCKETS: 'BUCKETS'; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index a4b233ad83..b3dded5ec7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -267,6 +267,9 @@ TOK_ALTER_MATERIALIZED_VIEW_REBUILD; TOK_REWRITE_ENABLED; TOK_REWRITE_DISABLED; TOK_VIEWPARTCOLS; +TOK_VIEWCLUSTERCOLS; +TOK_VIEWDISTRIBUTECOLS; +TOK_VIEWSORTCOLS; TOK_EXPLAIN; TOK_EXPLAIN_SQ_REWRITE; TOK_TABLESERIALIZER; @@ -1992,6 +1995,41 @@ viewPartition -> ^(TOK_VIEWPARTCOLS columnNameList) ; +viewOrganization +@init { pushMsg("view organization specification", state); } +@after { popMsg(state); } + : viewClusterSpec + | viewComplexSpec + ; + +viewClusterSpec +@init { pushMsg("view cluster specification", state); } +@after { popMsg(state); } + : KW_CLUSTERED KW_ON LPAREN columnNameList RPAREN + -> ^(TOK_VIEWCLUSTERCOLS columnNameList) + ; + +viewComplexSpec +@init { pushMsg("view complex specification", state); } +@after { popMsg(state); } + : viewSortSpec + | viewDistSpec viewSortSpec? + ; + +viewDistSpec +@init { pushMsg("view distribute specification", state); } +@after { popMsg(state); } + : KW_DISTRIBUTED KW_ON LPAREN colList=columnNameList RPAREN + -> ^(TOK_VIEWDISTRIBUTECOLS $colList) + ; + +viewSortSpec +@init { pushMsg("view sort specification", state); } +@after { popMsg(state); } + : KW_SORTED KW_ON LPAREN colList=columnNameList RPAREN + -> ^(TOK_VIEWSORTCOLS $colList) + ; + dropViewStatement @init { pushMsg("drop view statement", state); } @after { popMsg(state); } @@ -2004,7 +2042,8 @@ createMaterializedViewStatement } @after { popMsg(state); } : KW_CREATE KW_MATERIALIZED KW_VIEW (ifNotExists)? name=tableName - rewriteDisabled? tableComment? viewPartition? tableRowFormat? tableFileFormat? tableLocation? + rewriteDisabled? tableComment? viewPartition? viewOrganization? + tableRowFormat? tableFileFormat? tableLocation? tablePropertiesPrefixed? KW_AS selectStatementWithCTE -> ^(TOK_CREATE_MATERIALIZED_VIEW $name ifNotExists? @@ -2014,6 +2053,7 @@ createMaterializedViewStatement tableFileFormat? tableLocation? viewPartition? + viewOrganization? tablePropertiesPrefixed? selectStatementWithCTE ) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 013079c3d2..1935d3f3fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -824,7 +824,7 @@ nonReserved | KW_CASCADE | KW_CBO | KW_CHANGE | KW_CHECK | KW_CLUSTER | KW_CLUSTERED | KW_CLUSTERSTATUS | KW_COLLECTION | KW_COLUMNS | KW_COMMENT | KW_COMPACT | KW_COMPACTIONS | KW_COMPUTE | KW_CONCATENATE | KW_CONTINUE | KW_COST | KW_DATA | KW_DAY | KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY - | KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE | KW_DOW | KW_ELEM_TYPE + | KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE | KW_DISTRIBUTED | KW_DOW | KW_ELEM_TYPE | KW_ENABLE | KW_ENFORCED | KW_ESCAPED | KW_EXCLUSIVE | KW_EXPLAIN | KW_EXPORT | KW_FIELDS | KW_FILE | KW_FILEFORMAT | KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOLD_DDLTIME | KW_HOUR | KW_IDXPROPERTIES | KW_IGNORE | KW_INDEX | KW_INDEXES | KW_INPATH | KW_INPUTDRIVER | KW_INPUTFORMAT | KW_ITEMS | KW_JAR | KW_JOINCOST | KW_KILL diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e09fc379f5..dcfce38296 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; @@ -134,6 +135,7 @@ import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.UnionOperator; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField; import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.ReadEntity; @@ -1685,8 +1687,7 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan case HiveParser.TOK_DISTRIBUTEBY: // Get the distribute by aliases - these are aliased to the entries in - // the - // select list + // the select list queryProperties.setHasDistributeBy(true); qbp.setDistributeByExprForClause(ctx_1.dest, ast); if (qbp.getClusterByForClause(ctx_1.dest) != null) { @@ -4787,6 +4788,7 @@ public RowResolver handleInsertStatementSpec(List col_list, String col_list.addAll(new_col_list); return newOutputRR; } + String recommendName(ExprNodeDesc exp, String colAlias) { if (!colAlias.startsWith(autogenColAliasPrfxLbl)) { return null; @@ -6957,7 +6959,143 @@ private void genPartnCols(String dest, Operator input, QB qb, } } - @SuppressWarnings("unchecked") + private Operator genMaterializedViewDataOrgPlan(Table destinationTable, String sortColsStr, String distributeColsStr, + RowResolver inputRR, Operator input) throws SemanticException { + Map colNameToIdx = new HashMap<>(); + for (int i = 0; i < destinationTable.getCols().size(); i++) { + colNameToIdx.put(destinationTable.getCols().get(i).getName(), i); + } + List colInfos = inputRR.getColumnInfos(); + List sortColInfos = new ArrayList<>(); + if (sortColsStr != null) { + Utilities.decodeColumnNames(sortColsStr) + .forEach(s -> sortColInfos.add(colInfos.get(colNameToIdx.get(s)))); + } + List distributeColInfos = new ArrayList<>(); + if (distributeColsStr != null) { + Utilities.decodeColumnNames(distributeColsStr) + .forEach(s -> distributeColInfos.add(colInfos.get(colNameToIdx.get(s)))); + } + return genMaterializedViewDataOrgPlan(sortColInfos, distributeColInfos, inputRR, input); + } + + private Operator genMaterializedViewDataOrgPlan(List sortColInfos, List distributeColInfos, + RowResolver inputRR, Operator input) { + // In this case, we will introduce a RS and immediately after a SEL that restores + // the row schema to what follow-up operations are expecting + Set keys = sortColInfos.stream() + .map(ColumnInfo::getInternalName) + .collect(Collectors.toSet()); + Set distributeKeys = distributeColInfos.stream() + .map(ColumnInfo::getInternalName) + .collect(Collectors.toSet()); + ArrayList keyCols = new ArrayList<>(); + ArrayList keyColNames = new ArrayList<>(); + StringBuilder order = new StringBuilder(); + StringBuilder nullOrder = new StringBuilder(); + ArrayList valCols = new ArrayList<>(); + ArrayList valColNames = new ArrayList<>(); + ArrayList partCols = new ArrayList<>(); + for (ColumnInfo ci : inputRR.getRowSchema().getSignature()) { + ExprNodeColumnDesc e = new ExprNodeColumnDesc(ci); + String columnName = ci.getInternalName(); + if (keys.contains(columnName)) { + // key (sort column) + keyColNames.add(columnName); + keyCols.add(e); + order.append("+"); + nullOrder.append("a"); + } else { + // value + valColNames.add(columnName); + valCols.add(e); + } + if (distributeKeys.contains(columnName)) { + // distribute column + partCols.add(e.clone()); + } + } + // map _col0 to KEY._col0, etc + Map colExprMap = new HashMap<>(); + Map nameMapping = new HashMap<>(); + for (ExprNodeDesc keyCol : keyCols) { + String keyColName = keyCol.getExprString(); + keyColNames.add(keyColName); + colExprMap.put(Utilities.ReduceField.KEY + "." + keyColName, keyCol); + nameMapping.put(keyColName, Utilities.ReduceField.KEY + "." + keyColName); + } + for (ExprNodeDesc valCol : valCols) { + String colName = valCol.getExprString(); + valColNames.add(colName); + colExprMap.put(Utilities.ReduceField.VALUE + "." + colName, valCol); + nameMapping.put(colName, Utilities.ReduceField.VALUE + "." + colName); + } + // Create Key/Value TableDesc. When the operator plan is split into MR tasks, + // the reduce operator will initialize Extract operator with information + // from Key and Value TableDesc + List fields = PlanUtils.getFieldSchemasFromColumnList(keyCols, + keyColNames, 0, ""); + TableDesc keyTable = PlanUtils.getReduceKeyTableDesc(fields, order.toString(), nullOrder.toString()); + List valFields = PlanUtils.getFieldSchemasFromColumnList(valCols, + valColNames, 0, ""); + TableDesc valueTable = PlanUtils.getReduceValueTableDesc(valFields); + List> distinctColumnIndices = new ArrayList<>(); + // Number of reducers is set to default (-1) + ReduceSinkDesc rsConf = new ReduceSinkDesc(keyCols, keyCols.size(), valCols, + keyColNames, distinctColumnIndices, valColNames, -1, partCols, -1, keyTable, + valueTable, Operation.NOT_ACID); + RowResolver rsRR = new RowResolver(); + ArrayList rsSignature = new ArrayList<>(); + for (int index = 0; index < input.getSchema().getSignature().size(); index++) { + ColumnInfo colInfo = new ColumnInfo(input.getSchema().getSignature().get(index)); + String[] nm = inputRR.reverseLookup(colInfo.getInternalName()); + String[] nm2 = inputRR.getAlternateMappings(colInfo.getInternalName()); + colInfo.setInternalName(nameMapping.get(colInfo.getInternalName())); + rsSignature.add(colInfo); + rsRR.put(nm[0], nm[1], colInfo); + if (nm2 != null) { + rsRR.addMappingOnly(nm2[0], nm2[1], colInfo); + } + } + Operator result = putOpInsertMap(OperatorFactory.getAndMakeChild( + rsConf, new RowSchema(rsSignature), input), rsRR); + result.setColumnExprMap(colExprMap); + + // Create SEL operator + RowResolver selRR = new RowResolver(); + ArrayList selSignature = new ArrayList<>(); + List columnExprs = new ArrayList<>(); + List colNames = new ArrayList<>(); + Map selColExprMap = new HashMap<>(); + for (int index = 0; index < input.getSchema().getSignature().size(); index++) { + ColumnInfo colInfo = new ColumnInfo(input.getSchema().getSignature().get(index)); + String[] nm = inputRR.reverseLookup(colInfo.getInternalName()); + String[] nm2 = inputRR.getAlternateMappings(colInfo.getInternalName()); + selSignature.add(colInfo); + selRR.put(nm[0], nm[1], colInfo); + if (nm2 != null) { + selRR.addMappingOnly(nm2[0], nm2[1], colInfo); + } + String colName = colInfo.getInternalName(); + ExprNodeDesc exprNodeDesc; + if (keys.contains(colName)) { + exprNodeDesc = new ExprNodeColumnDesc(colInfo.getType(), ReduceField.KEY.toString() + "." + colName, null, false); + columnExprs.add(exprNodeDesc); + } else { + exprNodeDesc = new ExprNodeColumnDesc(colInfo.getType(), ReduceField.VALUE.toString() + "." + colName, null, false); + columnExprs.add(exprNodeDesc); + } + colNames.add(colName); + selColExprMap.put(colName, exprNodeDesc); + } + SelectDesc selConf = new SelectDesc(columnExprs, colNames); + result = putOpInsertMap(OperatorFactory.getAndMakeChild( + selConf, new RowSchema(selSignature), result), selRR); + result.setColumnExprMap(selColExprMap); + + return result; + } + private void setStatsForNonNativeTable(String dbName, String tableName) throws SemanticException { String qTableName = DDLSemanticAnalyzer.getDotName(new String[] { dbName, tableName }); @@ -7297,8 +7435,19 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // Add NOT NULL constraint check input = genConstraintsPlan(dest, qb, input); - // Add sorting/bucketing if needed - input = genBucketingSortingDest(dest, input, qb, tableDescriptor, destinationTable, rsCtx); + if (destinationTable.isMaterializedView() && + mvRebuildMode == MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD) { + // Data organization (DISTRIBUTED, SORTED, CLUSTERED) for materialized view + // TODO: We only do this for a full rebuild + String sortColsStr = destinationTable.getProperty(Constants.MATERIALIZED_VIEW_SORT_COLUMNS); + String distributeColsStr = destinationTable.getProperty(Constants.MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS); + if (sortColsStr != null || distributeColsStr != null) { + input = genMaterializedViewDataOrgPlan(destinationTable, sortColsStr, distributeColsStr, inputRR, input); + } + } else { + // Add sorting/bucketing if needed + input = genBucketingSortingDest(dest, input, qb, tableDescriptor, destinationTable, rsCtx); + } idToTableNameMap.put(String.valueOf(destTableId), destinationTable.getTableName()); currentTableId = destTableId; @@ -7414,8 +7563,19 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // Add NOT NULL constraint check input = genConstraintsPlan(dest, qb, input); - // Add sorting/bucketing if needed - input = genBucketingSortingDest(dest, input, qb, tableDescriptor, destinationTable, rsCtx); + if (destinationTable.isMaterializedView() && + mvRebuildMode == MaterializationRebuildMode.INSERT_OVERWRITE_REBUILD) { + // Data organization (DISTRIBUTED, SORTED, CLUSTERED) for materialized view + // TODO: We only do this for a full rebuild + String sortColsStr = destinationTable.getProperty(Constants.MATERIALIZED_VIEW_SORT_COLUMNS); + String distributeColsStr = destinationTable.getProperty(Constants.MATERIALIZED_VIEW_DISTRIBUTE_COLUMNS); + if (sortColsStr != null || distributeColsStr != null) { + input = genMaterializedViewDataOrgPlan(destinationTable, sortColsStr, distributeColsStr, inputRR, input); + } + } else { + // Add sorting/bucketing if needed + input = genBucketingSortingDest(dest, input, qb, tableDescriptor, destinationTable, rsCtx); + } idToTableNameMap.put(String.valueOf(destTableId), destinationTable.getTableName()); currentTableId = destTableId; @@ -7479,7 +7639,13 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) List fieldSchemas = null; List partitionColumns = null; List partitionColumnNames = null; + List sortColumns = null; + List sortColumnNames = null; + List distributeColumns = null; + List distributeColumnNames = null; List fileSinkColInfos = null; + List sortColInfos = null; + List distributeColInfos = null; CreateTableDesc tblDesc = qb.getTableDesc(); CreateViewDesc viewDesc = qb.getViewDesc(); if (tblDesc != null) { @@ -7518,7 +7684,13 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) fieldSchemas = new ArrayList<>(); partitionColumns = new ArrayList<>(); partitionColumnNames = viewDesc.getPartColNames(); + sortColumns = new ArrayList<>(); + sortColumnNames = viewDesc.getSortColNames(); + distributeColumns = new ArrayList<>(); + distributeColumnNames = viewDesc.getDistributeColNames(); fileSinkColInfos = new ArrayList<>(); + sortColInfos = new ArrayList<>(); + distributeColInfos = new ArrayList<>(); destTableIsTemporary = false; destTableIsMaterialization = false; } @@ -7552,8 +7724,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) } if (!CollectionUtils.isEmpty(partitionColumnNames)) { ColsAndTypes ct = deriveFileSinkColTypes( - inputRR, partitionColumnNames, fieldSchemas, partitionColumns, - fileSinkColInfos); + inputRR, partitionColumnNames, sortColumnNames, distributeColumnNames, fieldSchemas, partitionColumns, + sortColumns, distributeColumns, fileSinkColInfos, sortColInfos, distributeColInfos); cols = ct.cols; colTypes = ct.colTypes; dpCtx = new DynamicPartitionCtx(partitionColumnNames, @@ -7564,7 +7736,9 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) dpCtx.setRootPath(queryTmpdir); isPartitioned = true; } else { - ColsAndTypes ct = deriveFileSinkColTypes(inputRR, fieldSchemas); + ColsAndTypes ct = deriveFileSinkColTypes( + inputRR, sortColumnNames, distributeColumnNames, fieldSchemas, sortColumns, distributeColumns, + sortColInfos, distributeColInfos); cols = ct.cols; colTypes = ct.colTypes; isPartitioned = false; @@ -7577,6 +7751,10 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) } else if (viewDesc != null) { viewDesc.setSchema(new ArrayList<>(fieldSchemas)); viewDesc.setPartCols(new ArrayList<>(partitionColumns)); + if (viewDesc.isOrganized()) { + viewDesc.setSortCols(new ArrayList<>(sortColumns)); + viewDesc.setDistributeCols(new ArrayList<>(distributeColumns)); + } } boolean isDestTempFile = true; @@ -7633,6 +7811,11 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destTableIsFullAcid = AcidUtils.isFullAcidTable(destinationTable); + // Data organization (DISTRIBUTED, SORTED, CLUSTERED) for materialized view + if (viewDesc != null && viewDesc.isOrganized()) { + input = genMaterializedViewDataOrgPlan(sortColInfos, distributeColInfos, inputRR, input); + } + if (isPartitioned) { // Create a SELECT that may reorder the columns if needed RowResolver rowResolver = new RowResolver(); @@ -7654,7 +7837,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) input.setColumnExprMap(colExprMap); // If this is a partitioned CTAS or MV statement, we are going to create a LoadTableDesc - // object. Although the table does not exist in metastore, we will swamp the CreateTableTask + // object. Although the table does not exist in metastore, we will swap the CreateTableTask // and MoveTask resulting from this LoadTable so in this specific case, first we create // the metastore table, then we move and commit the partitions. At least for the time being, // this order needs to be enforced because metastore expects a table to exist before we can @@ -7817,20 +8000,25 @@ private boolean hasSetBatchSerializer(String serdeClassName) { serdeClassName.equalsIgnoreCase(ArrowColumnarBatchSerDe.class.getName()); } - private ColsAndTypes deriveFileSinkColTypes(RowResolver inputRR, List field_schemas) - throws SemanticException { - return deriveFileSinkColTypes(inputRR, new ArrayList<>(), field_schemas, new ArrayList<>(), new ArrayList<>()); + private ColsAndTypes deriveFileSinkColTypes(RowResolver inputRR, List sortColumnNames, List distributeColumnNames, + List fieldSchemas, List sortColumns, List distributeColumns, + List sortColInfos, List distributeColInfos) throws SemanticException { + return deriveFileSinkColTypes(inputRR, new ArrayList<>(), sortColumnNames, distributeColumnNames, + fieldSchemas, new ArrayList<>(), sortColumns, distributeColumns, new ArrayList<>(), + sortColInfos, distributeColInfos); } private ColsAndTypes deriveFileSinkColTypes( - RowResolver inputRR, List partitionColumnNames, - List columns, List partitionColumns, - List fileSinkColInfos) throws SemanticException { + RowResolver inputRR, List partitionColumnNames, List sortColumnNames, List distributeColumnNames, + List columns, List partitionColumns, List sortColumns, List distributeColumns, + List fileSinkColInfos, List sortColInfos, List distributeColInfos) throws SemanticException { ColsAndTypes result = new ColsAndTypes("", ""); List allColumns = new ArrayList<>(); List colInfos = inputRR.getColumnInfos(); List nonPartColInfos = new ArrayList<>(); SortedMap> partColInfos = new TreeMap<>(); + SortedMap> sColInfos = new TreeMap<>(); + SortedMap> dColInfos = new TreeMap<>(); boolean first = true; int numNonPartitionedCols = colInfos.size() - partitionColumnNames.size(); if (numNonPartitionedCols <= 0) { @@ -7865,6 +8053,18 @@ private ColsAndTypes deriveFileSinkColTypes( partColInfos.put(idx, Pair.of(col, colInfo)); isPartitionCol = true; } else { + if (sortColumnNames != null) { + idx = sortColumnNames.indexOf(colName); + if (idx >= 0) { + sColInfos.put(idx, Pair.of(col, colInfo)); + } + } + if (distributeColumnNames != null) { + idx = distributeColumnNames.indexOf(colName); + if (idx >= 0) { + dColInfos.put(idx, Pair.of(col, colInfo)); + } + } columns.add(col); nonPartColInfos.add(colInfo); } @@ -7907,10 +8107,33 @@ private ColsAndTypes deriveFileSinkColTypes( "Partition columns: " + partitionColumnNames); } + if (sortColumnNames != null && sColInfos.size() != sortColumnNames.size()) { + throw new SemanticException("Table declaration contains cluster/sort columns that are not present " + + "in query result schema. " + + "Query columns: " + allColumns + ". " + + "Organization columns: " + sortColumnNames); + } + + if (distributeColumnNames != null && dColInfos.size() != distributeColumnNames.size()) { + throw new SemanticException("Table declaration contains cluster/distribute columns that are not present " + + "in query result schema. " + + "Query columns: " + allColumns + ". " + + "Organization columns: " + distributeColumnNames); + } + // FileSinkColInfos comprise nonPartCols followed by partCols fileSinkColInfos.addAll(nonPartColInfos); partitionColumns.addAll(partColInfos.values().stream().map(Pair::getLeft).collect(Collectors.toList())); fileSinkColInfos.addAll(partColInfos.values().stream().map(Pair::getRight).collect(Collectors.toList())); + // data org columns + if (sortColumnNames != null) { + sortColumns.addAll(sColInfos.values().stream().map(Pair::getLeft).collect(Collectors.toList())); + sortColInfos.addAll(sColInfos.values().stream().map(Pair::getRight).collect(Collectors.toList())); + } + if (distributeColumnNames != null) { + distributeColumns.addAll(dColInfos.values().stream().map(Pair::getLeft).collect(Collectors.toList())); + distributeColInfos.addAll(dColInfos.values().stream().map(Pair::getRight).collect(Collectors.toList())); + } return result; } @@ -13717,6 +13940,8 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt ASTNode selectStmt = null; Map tblProps = null; List partColNames = null; + List sortColNames = null; + List distributeColNames = null; boolean isMaterialized = ast.getToken().getType() == HiveParser.TOK_CREATE_MATERIALIZED_VIEW; boolean isRebuild = false; String location = null; @@ -13760,6 +13985,19 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt case HiveParser.TOK_VIEWPARTCOLS: partColNames = getColumnNames((ASTNode) child.getChild(0)); break; + case HiveParser.TOK_VIEWCLUSTERCOLS: + assert sortColNames == null && distributeColNames == null; + sortColNames = getColumnNames((ASTNode) child.getChild(0)); + distributeColNames = new ArrayList<>(sortColNames); + break; + case HiveParser.TOK_VIEWSORTCOLS: + assert sortColNames == null; + sortColNames = getColumnNames((ASTNode) child.getChild(0)); + break; + case HiveParser.TOK_VIEWDISTRIBUTECOLS: + assert distributeColNames == null; + distributeColNames = getColumnNames((ASTNode) child.getChild(0)); + break; case HiveParser.TOK_TABLEROWFORMAT: rowFormatParams.analyzeRowFormat(child); break; @@ -13810,7 +14048,7 @@ protected ASTNode analyzeCreateView(ASTNode ast, QB qb, PlannerContext plannerCt if (isMaterialized) { createVwDesc = new CreateViewDesc( - dbDotTable, cols, comment, tblProps, partColNames, + dbDotTable, cols, comment, tblProps, partColNames, sortColNames, distributeColNames, ifNotExists, isRebuild, rewriteEnabled, isAlterViewAs, storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, storageFormat.getSerde(), storageFormat.getStorageHandler(), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java index 73a7d33690..6e2754ae1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java @@ -24,6 +24,7 @@ import java.util.Map; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -101,6 +102,8 @@ public ImportTableDesc(String dbName, Table table) throws Exception { null, // comment passed as table params table.getParameters(), table.getPartColNames(), + null, // sort columns passed as table params (if present) + null, // distribute columns passed as table params (if present) false,false,false,false, table.getSd().getInputFormat(), table.getSd().getOutputFormat(), diff --git a/ql/src/test/queries/clientpositive/materialized_view_cluster.q b/ql/src/test/queries/clientpositive/materialized_view_cluster.q new file mode 100644 index 0000000000..c9d3058051 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_cluster.q @@ -0,0 +1,98 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +EXPLAIN +CREATE MATERIALIZED VIEW cluster_mv_1 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +CREATE MATERIALIZED VIEW cluster_mv_1 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED cluster_mv_1; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_1/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_1/*; + +EXPLAIN +SELECT * FROM cluster_mv_1 where key = 238; + +SELECT * FROM cluster_mv_1 where key = 238; + +CREATE MATERIALIZED VIEW cluster_mv_2 CLUSTERED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_2/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_2/*; + +EXPLAIN +SELECT * FROM cluster_mv_2 where value = 'val_238'; + +SELECT * FROM cluster_mv_2 where value = 'val_238'; + +EXPLAIN +SELECT value FROM cluster_mv_2 where key = 238; + +SELECT value FROM cluster_mv_2 where key = 238; + +INSERT INTO src_txn VALUES (238, 'val_238_n'); + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD; + +SELECT * FROM cluster_mv_1 where key = 238; + +SELECT * FROM cluster_mv_2 where key = 238; + +CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +CREATE MATERIALIZED VIEW cluster_mv_3 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_3/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_3/*; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +SELECT * FROM cluster_mv_3 where key = 238; + +CREATE MATERIALIZED VIEW cluster_mv_4 CLUSTERED ON (key,`tes"t`,`te*#"s"t`) STORED AS TEXTFILE AS +SELECT value, key, key+1 as `tes"t`, key+2 as `te*#"s"t` FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED cluster_mv_4; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +set hive.materializedview.rebuild.incremental=false; + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +SELECT * FROM cluster_mv_3 where key = 238; + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD; + +SELECT * FROM cluster_mv_4 where key = 238; + +DESCRIBE FORMATTED cluster_mv_4; diff --git a/ql/src/test/queries/clientpositive/materialized_view_distribute.q b/ql/src/test/queries/clientpositive/materialized_view_distribute.q new file mode 100644 index 0000000000..ec8f5fe0b7 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_distribute.q @@ -0,0 +1,73 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +EXPLAIN +CREATE MATERIALIZED VIEW distribute_mv_1 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +CREATE MATERIALIZED VIEW distribute_mv_1 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED distribute_mv_1; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/distribute_mv_1/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/distribute_mv_1/*; + +EXPLAIN +SELECT * FROM distribute_mv_1 where key = 238; + +SELECT * FROM distribute_mv_1 where key = 238; + +CREATE MATERIALIZED VIEW distribute_mv_2 DISTRIBUTED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/distribute_mv_2/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/distribute_mv_2/*; + +EXPLAIN +SELECT * FROM distribute_mv_2 where value = 'val_238'; + +SELECT * FROM distribute_mv_2 where value = 'val_238'; + +EXPLAIN +SELECT value FROM distribute_mv_2 where key = 238; + +SELECT value FROM distribute_mv_2 where key = 238; + +INSERT INTO src_txn VALUES (238, 'val_238_n'); + +EXPLAIN +ALTER MATERIALIZED VIEW distribute_mv_1 REBUILD; + +ALTER MATERIALIZED VIEW distribute_mv_1 REBUILD; + +SELECT * FROM distribute_mv_1 where key = 238; + +SELECT * FROM distribute_mv_2 where key = 238; + +CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +CREATE MATERIALIZED VIEW distribute_mv_3 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/distribute_mv_3/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/distribute_mv_3/*; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +EXPLAIN +ALTER MATERIALIZED VIEW distribute_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW distribute_mv_3 REBUILD; + +SELECT * FROM distribute_mv_3 where key = 238; diff --git a/ql/src/test/queries/clientpositive/materialized_view_partition_cluster.q b/ql/src/test/queries/clientpositive/materialized_view_partition_cluster.q new file mode 100644 index 0000000000..a079605e03 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_partition_cluster.q @@ -0,0 +1,95 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +EXPLAIN +CREATE MATERIALIZED VIEW cluster_mv_1 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250; + +CREATE MATERIALIZED VIEW cluster_mv_1 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED cluster_mv_1; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_1/; + +EXPLAIN +SELECT * FROM cluster_mv_1 where key = 238; + +SELECT * FROM cluster_mv_1 where key = 238; + +CREATE MATERIALIZED VIEW cluster_mv_2 PARTITIONED ON (partkey) CLUSTERED ON (value) AS +SELECT key, value, key + 100 as partkey FROM src_txn where key > 200 and key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_2/; + +EXPLAIN +SELECT * FROM cluster_mv_2 where value = 'val_238'; + +SELECT * FROM cluster_mv_2 where value = 'val_238'; + +EXPLAIN +SELECT value FROM cluster_mv_2 where key = 238; + +SELECT value FROM cluster_mv_2 where key = 238; + +INSERT INTO src_txn VALUES (238, 'val_238_n'); + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD; + +SELECT * FROM cluster_mv_1 where key = 238; + +SELECT * FROM cluster_mv_2 where key = 238; + +CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +CREATE MATERIALIZED VIEW cluster_mv_3 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT src_txn.key + 100 as partkey, src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/cluster_mv_3/; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +SELECT * FROM cluster_mv_3 where key = 238; + +CREATE MATERIALIZED VIEW cluster_mv_4 PARTITIONED ON (partkey) CLUSTERED ON (key,`tes"t`,`te*#"s"t`) AS +SELECT src_txn.key + 100 as partkey, value, key, key+1 as `tes"t`, key+2 as `te*#"s"t` FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED cluster_mv_4; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +set hive.materializedview.rebuild.incremental=false; + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD; + +SELECT * FROM cluster_mv_3 where key = 238; + +EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD; + +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD; + +SELECT * FROM cluster_mv_4 where key = 238; + +DESCRIBE FORMATTED cluster_mv_4; diff --git a/ql/src/test/queries/clientpositive/materialized_view_sort.q b/ql/src/test/queries/clientpositive/materialized_view_sort.q new file mode 100644 index 0000000000..1c5acfa7c5 --- /dev/null +++ b/ql/src/test/queries/clientpositive/materialized_view_sort.q @@ -0,0 +1,73 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +EXPLAIN +CREATE MATERIALIZED VIEW sort_mv_1 SORTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +CREATE MATERIALIZED VIEW sort_mv_1 SORTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250; + +DESCRIBE FORMATTED sort_mv_1; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/sort_mv_1/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/sort_mv_1/*; + +EXPLAIN +SELECT * FROM sort_mv_1 where key = 238; + +SELECT * FROM sort_mv_1 where key = 238; + +CREATE MATERIALIZED VIEW sort_mv_2 SORTED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/sort_mv_2/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/sort_mv_2/*; + +EXPLAIN +SELECT * FROM sort_mv_2 where value = 'val_238'; + +SELECT * FROM sort_mv_2 where value = 'val_238'; + +EXPLAIN +SELECT value FROM sort_mv_2 where key = 238; + +SELECT value FROM sort_mv_2 where key = 238; + +INSERT INTO src_txn VALUES (238, 'val_238_n'); + +EXPLAIN +ALTER MATERIALIZED VIEW sort_mv_1 REBUILD; + +ALTER MATERIALIZED VIEW sort_mv_1 REBUILD; + +SELECT * FROM sort_mv_1 where key = 238; + +SELECT * FROM sort_mv_2 where key = 238; + +CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src; + +CREATE MATERIALIZED VIEW sort_mv_3 SORTED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250; + +dfs -ls ${system:test.warehouse.dir}/../localfs/warehouse/sort_mv_3/; +dfs -cat ${system:test.warehouse.dir}/../localfs/warehouse/sort_mv_3/*; + +INSERT INTO src_txn VALUES (238, 'val_238_n2'); + +EXPLAIN +ALTER MATERIALIZED VIEW sort_mv_3 REBUILD; + +ALTER MATERIALIZED VIEW sort_mv_3 REBUILD; + +SELECT * FROM sort_mv_3 where key = 238; diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out new file mode 100644 index 0000000000..86315f8afc --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_cluster.q.out @@ -0,0 +1,1351 @@ +PREHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn +POSTHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cluster_mv_1 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cluster_mv_1 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-0, Stage-2 + Stage-3 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: col1, col2 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') + minReductionHashAggr: 0.9818182 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View + distribute columns: key string + columns: value string, key string + sort columns: key string + expanded text: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 + name: default.cluster_mv_1 + original text: SELECT value, key FROM src_txn where key > 200 and key < 250 + rewrite enabled: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_1 + + Stage: Stage-5 + Materialized View Work + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_1 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_1 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_1 +PREHOOK: query: DESCRIBE FORMATTED cluster_mv_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cluster_mv_1 +POSTHOOK: query: DESCRIBE FORMATTED cluster_mv_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cluster_mv_1 +# col_name data_type comment +value string +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + materializedview.distribute.columns [\"key\"] + materializedview.sort.columns [\"key\"] + numFiles 2 + numRows 55 + rawDataSize 605 + totalSize 660 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT value, key FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +Rewrite Enabled: Yes +Outdated for Rewriting: No +Found 2 items +#### A masked pattern was here #### +val_201201 +val_203203 +val_203203 +val_205205 +val_205205 +val_209209 +val_209209 +val_213213 +val_213213 +val_218218 +val_221221 +val_221221 +val_223223 +val_223223 +val_226226 +val_233233 +val_233233 +val_237237 +val_237237 +val_239239 +val_239239 +val_241241 +val_242242 +val_242242 +val_247247 +val_248248 +val_202202 +val_207207 +val_207207 +val_208208 +val_208208 +val_208208 +val_214214 +val_216216 +val_216216 +val_217217 +val_217217 +val_219219 +val_219219 +val_222222 +val_224224 +val_224224 +val_228228 +val_229229 +val_229229 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_235235 +val_238238 +val_238238 +val_244244 +val_249249 +PREHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: cluster_mv_1 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_1 +#### A masked pattern was here #### +val_238 238 +val_238 238 +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_2 CLUSTERED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_2 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_2 CLUSTERED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_2 +Found 2 items +#### A masked pattern was here #### +202val_202 +203val_203 +203val_203 +205val_205 +205val_205 +213val_213 +213val_213 +214val_214 +216val_216 +216val_216 +218val_218 +221val_221 +221val_221 +228val_228 +229val_229 +229val_229 +233val_233 +233val_233 +235val_235 +237val_237 +237val_237 +238val_238 +238val_238 +239val_239 +239val_239 +241val_241 +248val_248 +201val_201 +207val_207 +207val_207 +208val_208 +208val_208 +208val_208 +209val_209 +209val_209 +217val_217 +217val_217 +219val_219 +219val_219 +222val_222 +223val_223 +223val_223 +224val_224 +224val_224 +226val_226 +230val_230 +230val_230 +230val_230 +230val_230 +230val_230 +242val_242 +242val_242 +244val_244 +247val_247 +249val_249 +PREHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: cluster_mv_2 + filterExpr: (value = 'val_238') (type: boolean) + Filter Operator + predicate: (value = 'val_238') (type: boolean) + Select Operator + expressions: key (type: string), 'val_238' (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM cluster_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: EXPLAIN +SELECT value FROM cluster_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT value FROM cluster_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: cluster_mv_2 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT value FROM cluster_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM cluster_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +val_238 +val_238 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@cluster_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9444444 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_1 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@cluster_mv_1 +POSTHOOK: Lineage: cluster_mv_1.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_1 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238_n 238 +PREHOOK: query: SELECT * FROM cluster_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn_2 +POSTHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn_2 +POSTHOOK: Lineage: src_txn_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_3 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_3 CLUSTERED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_3 +Found 1 items +#### A masked pattern was here #### +val_201201 +val_202202 +val_203203 +val_203203 +val_203203 +val_203203 +val_205205 +val_205205 +val_205205 +val_205205 +val_207207 +val_207207 +val_207207 +val_207207 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_209209 +val_209209 +val_209209 +val_209209 +val_213213 +val_213213 +val_213213 +val_213213 +val_214214 +val_216216 +val_216216 +val_216216 +val_216216 +val_217217 +val_217217 +val_217217 +val_217217 +val_218218 +val_219219 +val_219219 +val_219219 +val_219219 +val_221221 +val_221221 +val_221221 +val_221221 +val_222222 +val_223223 +val_223223 +val_223223 +val_223223 +val_224224 +val_224224 +val_224224 +val_224224 +val_226226 +val_228228 +val_229229 +val_229229 +val_229229 +val_229229 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_233233 +val_233233 +val_233233 +val_233233 +val_235235 +val_237237 +val_237237 +val_237237 +val_237237 +val_238238 +val_238238 +val_238238 +val_238238 +val_238_n238 +val_238_n238 +val_239239 +val_239239 +val_239239 +val_239239 +val_241241 +val_242242 +val_242242 +val_242242 +val_242242 +val_244244 +val_247247 +val_248248 +val_249249 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@cluster_mv_3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9444444 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@cluster_mv_3 +POSTHOOK: Lineage: cluster_mv_3.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_3 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238 238 +val_238 238 +val_238_n 238 +val_238_n 238 +val_238_n2 238 +val_238_n2 238 +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_4 CLUSTERED ON (key,`tes"t`,`te*#"s"t`) STORED AS TEXTFILE AS +SELECT value, key, key+1 as `tes"t`, key+2 as `te*#"s"t` FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_4 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_4 CLUSTERED ON (key,`tes"t`,`te*#"s"t`) STORED AS TEXTFILE AS +SELECT value, key, key+1 as `tes"t`, key+2 as `te*#"s"t` FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_4 +PREHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cluster_mv_4 +POSTHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cluster_mv_4 +# col_name data_type comment +value string +key string +tes"t double +te*#"s"t double + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"te*#\\\"s\\\"t\":\"true\",\"tes\\\"t\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + materializedview.distribute.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + materializedview.sort.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + numFiles 2 + numRows 57 + rawDataSize 1316 + totalSize 1373 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT value, key, key+1 as tes"t, key+2 as te*#"s"t FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `src_txn`.`value`, `src_txn`.`key`, `src_txn`.`key`+1 as `tes"t`, `src_txn`.`key`+2 as `te*#"s"t` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@cluster_mv_3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 503 Data size: 91043 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 5 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9818182 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@cluster_mv_3 +POSTHOOK: Lineage: cluster_mv_3.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_3 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238 238 +val_238 238 +val_238_n 238 +val_238_n 238 +val_238_n2 238 +val_238_n2 238 +val_238_n2 238 +val_238_n2 238 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_4 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@cluster_mv_4 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 503 Data size: 91043 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string), (UDFToDouble(key) + 1.0D) (type: double), (UDFToDouble(key) + 2.0D) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 55 Data size: 10835 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: double), _col3 (type: double) + sort order: +++ + Map-reduce partition columns: _col1 (type: string), _col2 (type: double), _col3 (type: double) + Statistics: Num rows: 55 Data size: 10835 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: double), KEY._col3 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 55 Data size: 10835 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 10835 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_4 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: double) + outputColumnNames: value, key, tes"t, te*#"s"t + Statistics: Num rows: 55 Data size: 10835 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll'), compute_stats(tes"t, 'hll'), compute_stats(te*#"s"t, 'hll') + minReductionHashAggr: 0.9818182 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct), _col2 (type: struct), _col3 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1760 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.cluster_mv_4 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key, tes"t, te*#"s"t + Column Types: string, string, double, double + Table: default.cluster_mv_4 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_4 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@cluster_mv_4 +POSTHOOK: Lineage: cluster_mv_4.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4.te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4.tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_4 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_4 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_4 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_4 +#### A masked pattern was here #### +val_238 238 239.0 240.0 +val_238 238 239.0 240.0 +val_238_n 238 239.0 240.0 +val_238_n2 238 239.0 240.0 +val_238_n2 238 239.0 240.0 +PREHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cluster_mv_4 +POSTHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cluster_mv_4 +# col_name data_type comment +value string +key string +tes"t double +te*#"s"t double + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"te*#\\\"s\\\"t\":\"true\",\"tes\\\"t\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + materializedview.distribute.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + materializedview.sort.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + numFiles 2 + numRows 58 + rawDataSize 1342 + totalSize 1400 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT value, key, key+1 as tes"t, key+2 as te*#"s"t FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `src_txn`.`value`, `src_txn`.`key`, `src_txn`.`key`+1 as `tes"t`, `src_txn`.`key`+2 as `te*#"s"t` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +Rewrite Enabled: Yes +Outdated for Rewriting: No diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_distribute.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_distribute.q.out new file mode 100644 index 0000000000..871bd3f772 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_distribute.q.out @@ -0,0 +1,909 @@ +PREHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn +POSTHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW distribute_mv_1 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@distribute_mv_1 +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW distribute_mv_1 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@distribute_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-0, Stage-2 + Stage-3 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.distribute_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: col1, col2 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') + minReductionHashAggr: 0.9818182 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View + distribute columns: key string + columns: value string, key string + expanded text: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 + name: default.distribute_mv_1 + original text: SELECT value, key FROM src_txn where key > 200 and key < 250 + rewrite enabled: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.distribute_mv_1 + + Stage: Stage-5 + Materialized View Work + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: CREATE MATERIALIZED VIEW distribute_mv_1 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@distribute_mv_1 +POSTHOOK: query: CREATE MATERIALIZED VIEW distribute_mv_1 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@distribute_mv_1 +PREHOOK: query: DESCRIBE FORMATTED distribute_mv_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@distribute_mv_1 +POSTHOOK: query: DESCRIBE FORMATTED distribute_mv_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@distribute_mv_1 +# col_name data_type comment +value string +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + materializedview.distribute.columns [\"key\"] + numFiles 1 + numRows 55 + rawDataSize 605 + totalSize 660 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT value, key FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +Rewrite Enabled: Yes +Outdated for Rewriting: No +Found 1 items +#### A masked pattern was here #### +val_238238 +val_224224 +val_213213 +val_209209 +val_219219 +val_237237 +val_207207 +val_208208 +val_247247 +val_203203 +val_205205 +val_221221 +val_208208 +val_239239 +val_213213 +val_216216 +val_221221 +val_241241 +val_230230 +val_217217 +val_230230 +val_208208 +val_230230 +val_205205 +val_238238 +val_224224 +val_242242 +val_242242 +val_226226 +val_229229 +val_233233 +val_223223 +val_218218 +val_228228 +val_209209 +val_230230 +val_216216 +val_219219 +val_239239 +val_223223 +val_244244 +val_202202 +val_229229 +val_235235 +val_203203 +val_201201 +val_217217 +val_237237 +val_248248 +val_230230 +val_207207 +val_249249 +val_214214 +val_233233 +val_222222 +PREHOOK: query: EXPLAIN +SELECT * FROM distribute_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM distribute_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: distribute_mv_1 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM distribute_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM distribute_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_1 +#### A masked pattern was here #### +val_238 238 +val_238 238 +PREHOOK: query: CREATE MATERIALIZED VIEW distribute_mv_2 DISTRIBUTED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@distribute_mv_2 +POSTHOOK: query: CREATE MATERIALIZED VIEW distribute_mv_2 DISTRIBUTED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@distribute_mv_2 +Found 1 items +#### A masked pattern was here #### +238val_238 +224val_224 +213val_213 +209val_209 +219val_219 +237val_237 +207val_207 +208val_208 +247val_247 +203val_203 +205val_205 +221val_221 +208val_208 +239val_239 +213val_213 +216val_216 +221val_221 +241val_241 +230val_230 +217val_217 +230val_230 +208val_208 +230val_230 +205val_205 +238val_238 +224val_224 +242val_242 +242val_242 +226val_226 +229val_229 +233val_233 +223val_223 +218val_218 +228val_228 +209val_209 +230val_230 +216val_216 +219val_219 +239val_239 +223val_223 +244val_244 +202val_202 +229val_229 +235val_235 +203val_203 +201val_201 +217val_217 +237val_237 +248val_248 +230val_230 +207val_207 +249val_249 +214val_214 +233val_233 +222val_222 +PREHOOK: query: EXPLAIN +SELECT * FROM distribute_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM distribute_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: distribute_mv_2 + filterExpr: (value = 'val_238') (type: boolean) + Filter Operator + predicate: (value = 'val_238') (type: boolean) + Select Operator + expressions: key (type: string), 'val_238' (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM distribute_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM distribute_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: EXPLAIN +SELECT value FROM distribute_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT value FROM distribute_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: distribute_mv_2 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT value FROM distribute_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM distribute_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +val_238 +val_238 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW distribute_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@distribute_mv_1 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW distribute_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@distribute_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.distribute_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9444444 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.distribute_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.distribute_mv_1 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW distribute_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@distribute_mv_1 +POSTHOOK: query: ALTER MATERIALIZED VIEW distribute_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@distribute_mv_1 +POSTHOOK: Lineage: distribute_mv_1.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: distribute_mv_1.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM distribute_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM distribute_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_1 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238_n 238 +PREHOOK: query: SELECT * FROM distribute_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM distribute_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_2 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn_2 +POSTHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn_2 +POSTHOOK: Lineage: src_txn_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE MATERIALIZED VIEW distribute_mv_3 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@distribute_mv_3 +POSTHOOK: query: CREATE MATERIALIZED VIEW distribute_mv_3 DISTRIBUTED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@distribute_mv_3 +Found 1 items +#### A masked pattern was here #### +val_201201 +val_202202 +val_203203 +val_203203 +val_203203 +val_203203 +val_205205 +val_205205 +val_205205 +val_205205 +val_207207 +val_207207 +val_207207 +val_207207 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_209209 +val_209209 +val_209209 +val_209209 +val_213213 +val_213213 +val_213213 +val_213213 +val_214214 +val_216216 +val_216216 +val_216216 +val_216216 +val_217217 +val_217217 +val_217217 +val_217217 +val_218218 +val_219219 +val_219219 +val_219219 +val_219219 +val_221221 +val_221221 +val_221221 +val_221221 +val_222222 +val_223223 +val_223223 +val_223223 +val_223223 +val_224224 +val_224224 +val_224224 +val_224224 +val_226226 +val_228228 +val_229229 +val_229229 +val_229229 +val_229229 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_233233 +val_233233 +val_233233 +val_233233 +val_235235 +val_237237 +val_237237 +val_237237 +val_237237 +val_238238 +val_238238 +val_238238 +val_238238 +val_238_n238 +val_238_n238 +val_239239 +val_239239 +val_239239 +val_239239 +val_241241 +val_242242 +val_242242 +val_242242 +val_242242 +val_244244 +val_247247 +val_248248 +val_249249 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW distribute_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@distribute_mv_3 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW distribute_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@distribute_mv_3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.distribute_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9444444 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.distribute_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.distribute_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW distribute_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@distribute_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW distribute_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@distribute_mv_3 +POSTHOOK: Lineage: distribute_mv_3.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: distribute_mv_3.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM distribute_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@distribute_mv_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM distribute_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@distribute_mv_3 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238 238 +val_238 238 +val_238_n 238 +val_238_n 238 +val_238_n2 238 +val_238_n2 238 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out new file mode 100644 index 0000000000..1f154d2b5a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out @@ -0,0 +1,2595 @@ +PREHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn +POSTHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cluster_mv_1 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_1 +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW cluster_mv_1 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-4 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-5 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string), (UDFToDouble(key) + 100.0D) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 10230 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 10230 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col2 (type: double) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string), VALUE._col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 10230 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 10230 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + outputColumnNames: col1, col2, col3 + Statistics: Num rows: 55 Data size: 10230 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') + keys: col3 (type: double) + minReductionHashAggr: 0.5090909 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View + distribute columns: key string + partition columns: partkey double + columns: value string, key string + sort columns: key string + expanded text: SELECT `value`, `key`, `partkey` FROM (SELECT `src_txn`.`value`, `src_txn`.`key`, `src_txn`.`key` + 100 as `partkey` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250) `cluster_mv_1` + name: default.cluster_mv_1 + original text: SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250 + rewrite enabled: true + + Stage: Stage-0 + Move Operator + tables: + partition: + partkey + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_1 + + Stage: Stage-5 + Materialized View Work + +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_1 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_1 +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_1 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_1 +POSTHOOK: Output: default@cluster_mv_1@partkey=301.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=302.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=303.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=305.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=307.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=308.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=309.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=313.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=314.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=316.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=317.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=318.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=319.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=321.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=322.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=323.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=324.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=326.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=328.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=329.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=330.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=333.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=335.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=337.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=338.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=339.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=341.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=342.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=344.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=347.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=348.0 +POSTHOOK: Output: default@cluster_mv_1@partkey=349.0 +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=301.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=301.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=302.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=302.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=303.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=303.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=305.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=305.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=307.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=307.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=308.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=308.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=309.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=309.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=313.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=313.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=314.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=314.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=316.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=316.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=317.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=317.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=318.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=318.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=319.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=319.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=321.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=321.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=322.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=322.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=323.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=323.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=324.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=324.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=326.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=326.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=328.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=328.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=329.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=329.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=330.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=330.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=333.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=333.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=335.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=335.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=337.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=337.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=339.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=339.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=341.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=341.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=342.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=342.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=344.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=344.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=347.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=347.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=348.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=348.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=349.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=349.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: DESCRIBE FORMATTED cluster_mv_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cluster_mv_1 +POSTHOOK: query: DESCRIBE FORMATTED cluster_mv_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cluster_mv_1 +# col_name data_type comment +value string +key string + +# Partition Information +# col_name data_type comment +partkey double + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + materializedview.distribute.columns [\"key\"] + materializedview.sort.columns [\"key\"] + numFiles 32 + numPartitions 32 + numRows 55 + rawDataSize 9790 + totalSize 10806 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT value, key, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `value`, `key`, `partkey` FROM (SELECT `src_txn`.`value`, `src_txn`.`key`, `src_txn`.`key` + 100 as `partkey` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250) `cluster_mv_1` +Rewrite Enabled: Yes +Outdated for Rewriting: No +Found 32 items +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_1 +PREHOOK: Input: default@cluster_mv_1@partkey=301.0 +PREHOOK: Input: default@cluster_mv_1@partkey=302.0 +PREHOOK: Input: default@cluster_mv_1@partkey=303.0 +PREHOOK: Input: default@cluster_mv_1@partkey=305.0 +PREHOOK: Input: default@cluster_mv_1@partkey=307.0 +PREHOOK: Input: default@cluster_mv_1@partkey=308.0 +PREHOOK: Input: default@cluster_mv_1@partkey=309.0 +PREHOOK: Input: default@cluster_mv_1@partkey=313.0 +PREHOOK: Input: default@cluster_mv_1@partkey=314.0 +PREHOOK: Input: default@cluster_mv_1@partkey=316.0 +PREHOOK: Input: default@cluster_mv_1@partkey=317.0 +PREHOOK: Input: default@cluster_mv_1@partkey=318.0 +PREHOOK: Input: default@cluster_mv_1@partkey=319.0 +PREHOOK: Input: default@cluster_mv_1@partkey=321.0 +PREHOOK: Input: default@cluster_mv_1@partkey=322.0 +PREHOOK: Input: default@cluster_mv_1@partkey=323.0 +PREHOOK: Input: default@cluster_mv_1@partkey=324.0 +PREHOOK: Input: default@cluster_mv_1@partkey=326.0 +PREHOOK: Input: default@cluster_mv_1@partkey=328.0 +PREHOOK: Input: default@cluster_mv_1@partkey=329.0 +PREHOOK: Input: default@cluster_mv_1@partkey=330.0 +PREHOOK: Input: default@cluster_mv_1@partkey=333.0 +PREHOOK: Input: default@cluster_mv_1@partkey=335.0 +PREHOOK: Input: default@cluster_mv_1@partkey=337.0 +PREHOOK: Input: default@cluster_mv_1@partkey=338.0 +PREHOOK: Input: default@cluster_mv_1@partkey=339.0 +PREHOOK: Input: default@cluster_mv_1@partkey=341.0 +PREHOOK: Input: default@cluster_mv_1@partkey=342.0 +PREHOOK: Input: default@cluster_mv_1@partkey=344.0 +PREHOOK: Input: default@cluster_mv_1@partkey=347.0 +PREHOOK: Input: default@cluster_mv_1@partkey=348.0 +PREHOOK: Input: default@cluster_mv_1@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_1 +POSTHOOK: Input: default@cluster_mv_1@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=349.0 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: cluster_mv_1 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string), key (type: string), partkey (type: double) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_1 +PREHOOK: Input: default@cluster_mv_1@partkey=301.0 +PREHOOK: Input: default@cluster_mv_1@partkey=302.0 +PREHOOK: Input: default@cluster_mv_1@partkey=303.0 +PREHOOK: Input: default@cluster_mv_1@partkey=305.0 +PREHOOK: Input: default@cluster_mv_1@partkey=307.0 +PREHOOK: Input: default@cluster_mv_1@partkey=308.0 +PREHOOK: Input: default@cluster_mv_1@partkey=309.0 +PREHOOK: Input: default@cluster_mv_1@partkey=313.0 +PREHOOK: Input: default@cluster_mv_1@partkey=314.0 +PREHOOK: Input: default@cluster_mv_1@partkey=316.0 +PREHOOK: Input: default@cluster_mv_1@partkey=317.0 +PREHOOK: Input: default@cluster_mv_1@partkey=318.0 +PREHOOK: Input: default@cluster_mv_1@partkey=319.0 +PREHOOK: Input: default@cluster_mv_1@partkey=321.0 +PREHOOK: Input: default@cluster_mv_1@partkey=322.0 +PREHOOK: Input: default@cluster_mv_1@partkey=323.0 +PREHOOK: Input: default@cluster_mv_1@partkey=324.0 +PREHOOK: Input: default@cluster_mv_1@partkey=326.0 +PREHOOK: Input: default@cluster_mv_1@partkey=328.0 +PREHOOK: Input: default@cluster_mv_1@partkey=329.0 +PREHOOK: Input: default@cluster_mv_1@partkey=330.0 +PREHOOK: Input: default@cluster_mv_1@partkey=333.0 +PREHOOK: Input: default@cluster_mv_1@partkey=335.0 +PREHOOK: Input: default@cluster_mv_1@partkey=337.0 +PREHOOK: Input: default@cluster_mv_1@partkey=338.0 +PREHOOK: Input: default@cluster_mv_1@partkey=339.0 +PREHOOK: Input: default@cluster_mv_1@partkey=341.0 +PREHOOK: Input: default@cluster_mv_1@partkey=342.0 +PREHOOK: Input: default@cluster_mv_1@partkey=344.0 +PREHOOK: Input: default@cluster_mv_1@partkey=347.0 +PREHOOK: Input: default@cluster_mv_1@partkey=348.0 +PREHOOK: Input: default@cluster_mv_1@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_1 +POSTHOOK: Input: default@cluster_mv_1@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=349.0 +#### A masked pattern was here #### +val_238 238 338.0 +val_238 238 338.0 +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_2 PARTITIONED ON (partkey) CLUSTERED ON (value) AS +SELECT key, value, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_2 +PREHOOK: Output: default@cluster_mv_2 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_2 PARTITIONED ON (partkey) CLUSTERED ON (value) AS +SELECT key, value, key + 100 as partkey FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_2 +POSTHOOK: Output: default@cluster_mv_2@partkey=301.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=302.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=303.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=305.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=307.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=308.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=309.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=313.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=314.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=316.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=317.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=318.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=319.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=321.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=322.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=323.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=324.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=326.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=328.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=329.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=330.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=333.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=335.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=337.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=338.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=339.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=341.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=342.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=344.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=347.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=348.0 +POSTHOOK: Output: default@cluster_mv_2@partkey=349.0 +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=301.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=301.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=302.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=302.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=303.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=303.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=305.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=305.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=307.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=307.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=308.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=308.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=309.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=309.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=313.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=313.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=314.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=314.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=316.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=316.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=317.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=317.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=318.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=318.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=319.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=319.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=321.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=321.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=322.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=322.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=323.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=323.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=324.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=324.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=326.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=326.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=328.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=328.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=329.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=329.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=330.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=330.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=333.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=333.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=335.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=335.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=337.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=337.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=339.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=339.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=341.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=341.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=342.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=342.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=344.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=344.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=347.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=347.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=348.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=348.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=349.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_2 PARTITION(partkey=349.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +Found 32 items +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +PREHOOK: Input: default@cluster_mv_2@partkey=301.0 +PREHOOK: Input: default@cluster_mv_2@partkey=302.0 +PREHOOK: Input: default@cluster_mv_2@partkey=303.0 +PREHOOK: Input: default@cluster_mv_2@partkey=305.0 +PREHOOK: Input: default@cluster_mv_2@partkey=307.0 +PREHOOK: Input: default@cluster_mv_2@partkey=308.0 +PREHOOK: Input: default@cluster_mv_2@partkey=309.0 +PREHOOK: Input: default@cluster_mv_2@partkey=313.0 +PREHOOK: Input: default@cluster_mv_2@partkey=314.0 +PREHOOK: Input: default@cluster_mv_2@partkey=316.0 +PREHOOK: Input: default@cluster_mv_2@partkey=317.0 +PREHOOK: Input: default@cluster_mv_2@partkey=318.0 +PREHOOK: Input: default@cluster_mv_2@partkey=319.0 +PREHOOK: Input: default@cluster_mv_2@partkey=321.0 +PREHOOK: Input: default@cluster_mv_2@partkey=322.0 +PREHOOK: Input: default@cluster_mv_2@partkey=323.0 +PREHOOK: Input: default@cluster_mv_2@partkey=324.0 +PREHOOK: Input: default@cluster_mv_2@partkey=326.0 +PREHOOK: Input: default@cluster_mv_2@partkey=328.0 +PREHOOK: Input: default@cluster_mv_2@partkey=329.0 +PREHOOK: Input: default@cluster_mv_2@partkey=330.0 +PREHOOK: Input: default@cluster_mv_2@partkey=333.0 +PREHOOK: Input: default@cluster_mv_2@partkey=335.0 +PREHOOK: Input: default@cluster_mv_2@partkey=337.0 +PREHOOK: Input: default@cluster_mv_2@partkey=338.0 +PREHOOK: Input: default@cluster_mv_2@partkey=339.0 +PREHOOK: Input: default@cluster_mv_2@partkey=341.0 +PREHOOK: Input: default@cluster_mv_2@partkey=342.0 +PREHOOK: Input: default@cluster_mv_2@partkey=344.0 +PREHOOK: Input: default@cluster_mv_2@partkey=347.0 +PREHOOK: Input: default@cluster_mv_2@partkey=348.0 +PREHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM cluster_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +POSTHOOK: Input: default@cluster_mv_2@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: cluster_mv_2 + filterExpr: (value = 'val_238') (type: boolean) + Filter Operator + predicate: (value = 'val_238') (type: boolean) + Select Operator + expressions: key (type: string), 'val_238' (type: string), partkey (type: double) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT * FROM cluster_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +PREHOOK: Input: default@cluster_mv_2@partkey=301.0 +PREHOOK: Input: default@cluster_mv_2@partkey=302.0 +PREHOOK: Input: default@cluster_mv_2@partkey=303.0 +PREHOOK: Input: default@cluster_mv_2@partkey=305.0 +PREHOOK: Input: default@cluster_mv_2@partkey=307.0 +PREHOOK: Input: default@cluster_mv_2@partkey=308.0 +PREHOOK: Input: default@cluster_mv_2@partkey=309.0 +PREHOOK: Input: default@cluster_mv_2@partkey=313.0 +PREHOOK: Input: default@cluster_mv_2@partkey=314.0 +PREHOOK: Input: default@cluster_mv_2@partkey=316.0 +PREHOOK: Input: default@cluster_mv_2@partkey=317.0 +PREHOOK: Input: default@cluster_mv_2@partkey=318.0 +PREHOOK: Input: default@cluster_mv_2@partkey=319.0 +PREHOOK: Input: default@cluster_mv_2@partkey=321.0 +PREHOOK: Input: default@cluster_mv_2@partkey=322.0 +PREHOOK: Input: default@cluster_mv_2@partkey=323.0 +PREHOOK: Input: default@cluster_mv_2@partkey=324.0 +PREHOOK: Input: default@cluster_mv_2@partkey=326.0 +PREHOOK: Input: default@cluster_mv_2@partkey=328.0 +PREHOOK: Input: default@cluster_mv_2@partkey=329.0 +PREHOOK: Input: default@cluster_mv_2@partkey=330.0 +PREHOOK: Input: default@cluster_mv_2@partkey=333.0 +PREHOOK: Input: default@cluster_mv_2@partkey=335.0 +PREHOOK: Input: default@cluster_mv_2@partkey=337.0 +PREHOOK: Input: default@cluster_mv_2@partkey=338.0 +PREHOOK: Input: default@cluster_mv_2@partkey=339.0 +PREHOOK: Input: default@cluster_mv_2@partkey=341.0 +PREHOOK: Input: default@cluster_mv_2@partkey=342.0 +PREHOOK: Input: default@cluster_mv_2@partkey=344.0 +PREHOOK: Input: default@cluster_mv_2@partkey=347.0 +PREHOOK: Input: default@cluster_mv_2@partkey=348.0 +PREHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +POSTHOOK: Input: default@cluster_mv_2@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +238 val_238 338.0 +238 val_238 338.0 +PREHOOK: query: EXPLAIN +SELECT value FROM cluster_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +PREHOOK: Input: default@cluster_mv_2@partkey=301.0 +PREHOOK: Input: default@cluster_mv_2@partkey=302.0 +PREHOOK: Input: default@cluster_mv_2@partkey=303.0 +PREHOOK: Input: default@cluster_mv_2@partkey=305.0 +PREHOOK: Input: default@cluster_mv_2@partkey=307.0 +PREHOOK: Input: default@cluster_mv_2@partkey=308.0 +PREHOOK: Input: default@cluster_mv_2@partkey=309.0 +PREHOOK: Input: default@cluster_mv_2@partkey=313.0 +PREHOOK: Input: default@cluster_mv_2@partkey=314.0 +PREHOOK: Input: default@cluster_mv_2@partkey=316.0 +PREHOOK: Input: default@cluster_mv_2@partkey=317.0 +PREHOOK: Input: default@cluster_mv_2@partkey=318.0 +PREHOOK: Input: default@cluster_mv_2@partkey=319.0 +PREHOOK: Input: default@cluster_mv_2@partkey=321.0 +PREHOOK: Input: default@cluster_mv_2@partkey=322.0 +PREHOOK: Input: default@cluster_mv_2@partkey=323.0 +PREHOOK: Input: default@cluster_mv_2@partkey=324.0 +PREHOOK: Input: default@cluster_mv_2@partkey=326.0 +PREHOOK: Input: default@cluster_mv_2@partkey=328.0 +PREHOOK: Input: default@cluster_mv_2@partkey=329.0 +PREHOOK: Input: default@cluster_mv_2@partkey=330.0 +PREHOOK: Input: default@cluster_mv_2@partkey=333.0 +PREHOOK: Input: default@cluster_mv_2@partkey=335.0 +PREHOOK: Input: default@cluster_mv_2@partkey=337.0 +PREHOOK: Input: default@cluster_mv_2@partkey=338.0 +PREHOOK: Input: default@cluster_mv_2@partkey=339.0 +PREHOOK: Input: default@cluster_mv_2@partkey=341.0 +PREHOOK: Input: default@cluster_mv_2@partkey=342.0 +PREHOOK: Input: default@cluster_mv_2@partkey=344.0 +PREHOOK: Input: default@cluster_mv_2@partkey=347.0 +PREHOOK: Input: default@cluster_mv_2@partkey=348.0 +PREHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT value FROM cluster_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +POSTHOOK: Input: default@cluster_mv_2@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: cluster_mv_2 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT value FROM cluster_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +PREHOOK: Input: default@cluster_mv_2@partkey=301.0 +PREHOOK: Input: default@cluster_mv_2@partkey=302.0 +PREHOOK: Input: default@cluster_mv_2@partkey=303.0 +PREHOOK: Input: default@cluster_mv_2@partkey=305.0 +PREHOOK: Input: default@cluster_mv_2@partkey=307.0 +PREHOOK: Input: default@cluster_mv_2@partkey=308.0 +PREHOOK: Input: default@cluster_mv_2@partkey=309.0 +PREHOOK: Input: default@cluster_mv_2@partkey=313.0 +PREHOOK: Input: default@cluster_mv_2@partkey=314.0 +PREHOOK: Input: default@cluster_mv_2@partkey=316.0 +PREHOOK: Input: default@cluster_mv_2@partkey=317.0 +PREHOOK: Input: default@cluster_mv_2@partkey=318.0 +PREHOOK: Input: default@cluster_mv_2@partkey=319.0 +PREHOOK: Input: default@cluster_mv_2@partkey=321.0 +PREHOOK: Input: default@cluster_mv_2@partkey=322.0 +PREHOOK: Input: default@cluster_mv_2@partkey=323.0 +PREHOOK: Input: default@cluster_mv_2@partkey=324.0 +PREHOOK: Input: default@cluster_mv_2@partkey=326.0 +PREHOOK: Input: default@cluster_mv_2@partkey=328.0 +PREHOOK: Input: default@cluster_mv_2@partkey=329.0 +PREHOOK: Input: default@cluster_mv_2@partkey=330.0 +PREHOOK: Input: default@cluster_mv_2@partkey=333.0 +PREHOOK: Input: default@cluster_mv_2@partkey=335.0 +PREHOOK: Input: default@cluster_mv_2@partkey=337.0 +PREHOOK: Input: default@cluster_mv_2@partkey=338.0 +PREHOOK: Input: default@cluster_mv_2@partkey=339.0 +PREHOOK: Input: default@cluster_mv_2@partkey=341.0 +PREHOOK: Input: default@cluster_mv_2@partkey=342.0 +PREHOOK: Input: default@cluster_mv_2@partkey=344.0 +PREHOOK: Input: default@cluster_mv_2@partkey=347.0 +PREHOOK: Input: default@cluster_mv_2@partkey=348.0 +PREHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM cluster_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +POSTHOOK: Input: default@cluster_mv_2@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +val_238 +val_238 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string), (UDFToDouble(key) + 100.0D) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 18 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + outputColumnNames: value, key, partkey + Statistics: Num rows: 18 Data size: 3384 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + keys: partkey (type: double) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + partkey + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_1 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_1 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@cluster_mv_1@partkey=338.0 +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_1 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_1 +PREHOOK: Input: default@cluster_mv_1@partkey=301.0 +PREHOOK: Input: default@cluster_mv_1@partkey=302.0 +PREHOOK: Input: default@cluster_mv_1@partkey=303.0 +PREHOOK: Input: default@cluster_mv_1@partkey=305.0 +PREHOOK: Input: default@cluster_mv_1@partkey=307.0 +PREHOOK: Input: default@cluster_mv_1@partkey=308.0 +PREHOOK: Input: default@cluster_mv_1@partkey=309.0 +PREHOOK: Input: default@cluster_mv_1@partkey=313.0 +PREHOOK: Input: default@cluster_mv_1@partkey=314.0 +PREHOOK: Input: default@cluster_mv_1@partkey=316.0 +PREHOOK: Input: default@cluster_mv_1@partkey=317.0 +PREHOOK: Input: default@cluster_mv_1@partkey=318.0 +PREHOOK: Input: default@cluster_mv_1@partkey=319.0 +PREHOOK: Input: default@cluster_mv_1@partkey=321.0 +PREHOOK: Input: default@cluster_mv_1@partkey=322.0 +PREHOOK: Input: default@cluster_mv_1@partkey=323.0 +PREHOOK: Input: default@cluster_mv_1@partkey=324.0 +PREHOOK: Input: default@cluster_mv_1@partkey=326.0 +PREHOOK: Input: default@cluster_mv_1@partkey=328.0 +PREHOOK: Input: default@cluster_mv_1@partkey=329.0 +PREHOOK: Input: default@cluster_mv_1@partkey=330.0 +PREHOOK: Input: default@cluster_mv_1@partkey=333.0 +PREHOOK: Input: default@cluster_mv_1@partkey=335.0 +PREHOOK: Input: default@cluster_mv_1@partkey=337.0 +PREHOOK: Input: default@cluster_mv_1@partkey=338.0 +PREHOOK: Input: default@cluster_mv_1@partkey=339.0 +PREHOOK: Input: default@cluster_mv_1@partkey=341.0 +PREHOOK: Input: default@cluster_mv_1@partkey=342.0 +PREHOOK: Input: default@cluster_mv_1@partkey=344.0 +PREHOOK: Input: default@cluster_mv_1@partkey=347.0 +PREHOOK: Input: default@cluster_mv_1@partkey=348.0 +PREHOOK: Input: default@cluster_mv_1@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_1 +POSTHOOK: Input: default@cluster_mv_1@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_1@partkey=349.0 +#### A masked pattern was here #### +val_238 238 338.0 +val_238 238 338.0 +val_238_n 238 338.0 +PREHOOK: query: SELECT * FROM cluster_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_2 +PREHOOK: Input: default@cluster_mv_2@partkey=301.0 +PREHOOK: Input: default@cluster_mv_2@partkey=302.0 +PREHOOK: Input: default@cluster_mv_2@partkey=303.0 +PREHOOK: Input: default@cluster_mv_2@partkey=305.0 +PREHOOK: Input: default@cluster_mv_2@partkey=307.0 +PREHOOK: Input: default@cluster_mv_2@partkey=308.0 +PREHOOK: Input: default@cluster_mv_2@partkey=309.0 +PREHOOK: Input: default@cluster_mv_2@partkey=313.0 +PREHOOK: Input: default@cluster_mv_2@partkey=314.0 +PREHOOK: Input: default@cluster_mv_2@partkey=316.0 +PREHOOK: Input: default@cluster_mv_2@partkey=317.0 +PREHOOK: Input: default@cluster_mv_2@partkey=318.0 +PREHOOK: Input: default@cluster_mv_2@partkey=319.0 +PREHOOK: Input: default@cluster_mv_2@partkey=321.0 +PREHOOK: Input: default@cluster_mv_2@partkey=322.0 +PREHOOK: Input: default@cluster_mv_2@partkey=323.0 +PREHOOK: Input: default@cluster_mv_2@partkey=324.0 +PREHOOK: Input: default@cluster_mv_2@partkey=326.0 +PREHOOK: Input: default@cluster_mv_2@partkey=328.0 +PREHOOK: Input: default@cluster_mv_2@partkey=329.0 +PREHOOK: Input: default@cluster_mv_2@partkey=330.0 +PREHOOK: Input: default@cluster_mv_2@partkey=333.0 +PREHOOK: Input: default@cluster_mv_2@partkey=335.0 +PREHOOK: Input: default@cluster_mv_2@partkey=337.0 +PREHOOK: Input: default@cluster_mv_2@partkey=338.0 +PREHOOK: Input: default@cluster_mv_2@partkey=339.0 +PREHOOK: Input: default@cluster_mv_2@partkey=341.0 +PREHOOK: Input: default@cluster_mv_2@partkey=342.0 +PREHOOK: Input: default@cluster_mv_2@partkey=344.0 +PREHOOK: Input: default@cluster_mv_2@partkey=347.0 +PREHOOK: Input: default@cluster_mv_2@partkey=348.0 +PREHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_2 +POSTHOOK: Input: default@cluster_mv_2@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_2@partkey=349.0 +#### A masked pattern was here #### +238 val_238 338.0 +238 val_238 338.0 +PREHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn_2 +POSTHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn_2 +POSTHOOK: Lineage: src_txn_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_3 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT src_txn.key + 100 as partkey, src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_3 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_3 PARTITIONED ON (partkey) CLUSTERED ON (key) AS +SELECT src_txn.key + 100 as partkey, src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_3 +POSTHOOK: Output: default@cluster_mv_3@partkey=301.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=302.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=303.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=305.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=307.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=308.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=309.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=313.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=314.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=316.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=317.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=318.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=319.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=321.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=322.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=323.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=324.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=326.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=328.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=329.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=330.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=333.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=335.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=337.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=338.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=339.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=341.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=342.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=344.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=347.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=348.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=349.0 +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=301.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=301.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=302.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=302.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=303.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=303.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=305.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=305.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=307.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=307.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=308.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=308.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=309.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=309.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=313.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=313.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=314.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=314.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=316.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=316.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=317.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=317.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=318.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=318.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=319.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=319.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=321.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=321.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=322.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=322.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=323.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=323.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=324.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=324.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=326.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=326.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=328.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=328.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=329.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=329.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=330.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=330.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=333.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=333.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=335.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=335.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=337.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=337.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=339.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=339.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=341.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=341.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=342.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=342.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=344.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=344.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=347.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=347.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=348.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=348.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=349.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=349.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +Found 32 items +#### A masked pattern was here #### +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), (UDFToDouble(key) + 100.0D) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: double) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + outputColumnNames: value, key, partkey + Statistics: Num rows: 18 Data size: 3402 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + keys: partkey (type: double) + minReductionHashAggr: 0.5 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 7992 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + partkey + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@cluster_mv_3@partkey=338.0 +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_3 +PREHOOK: Input: default@cluster_mv_3@partkey=301.0 +PREHOOK: Input: default@cluster_mv_3@partkey=302.0 +PREHOOK: Input: default@cluster_mv_3@partkey=303.0 +PREHOOK: Input: default@cluster_mv_3@partkey=305.0 +PREHOOK: Input: default@cluster_mv_3@partkey=307.0 +PREHOOK: Input: default@cluster_mv_3@partkey=308.0 +PREHOOK: Input: default@cluster_mv_3@partkey=309.0 +PREHOOK: Input: default@cluster_mv_3@partkey=313.0 +PREHOOK: Input: default@cluster_mv_3@partkey=314.0 +PREHOOK: Input: default@cluster_mv_3@partkey=316.0 +PREHOOK: Input: default@cluster_mv_3@partkey=317.0 +PREHOOK: Input: default@cluster_mv_3@partkey=318.0 +PREHOOK: Input: default@cluster_mv_3@partkey=319.0 +PREHOOK: Input: default@cluster_mv_3@partkey=321.0 +PREHOOK: Input: default@cluster_mv_3@partkey=322.0 +PREHOOK: Input: default@cluster_mv_3@partkey=323.0 +PREHOOK: Input: default@cluster_mv_3@partkey=324.0 +PREHOOK: Input: default@cluster_mv_3@partkey=326.0 +PREHOOK: Input: default@cluster_mv_3@partkey=328.0 +PREHOOK: Input: default@cluster_mv_3@partkey=329.0 +PREHOOK: Input: default@cluster_mv_3@partkey=330.0 +PREHOOK: Input: default@cluster_mv_3@partkey=333.0 +PREHOOK: Input: default@cluster_mv_3@partkey=335.0 +PREHOOK: Input: default@cluster_mv_3@partkey=337.0 +PREHOOK: Input: default@cluster_mv_3@partkey=338.0 +PREHOOK: Input: default@cluster_mv_3@partkey=339.0 +PREHOOK: Input: default@cluster_mv_3@partkey=341.0 +PREHOOK: Input: default@cluster_mv_3@partkey=342.0 +PREHOOK: Input: default@cluster_mv_3@partkey=344.0 +PREHOOK: Input: default@cluster_mv_3@partkey=347.0 +PREHOOK: Input: default@cluster_mv_3@partkey=348.0 +PREHOOK: Input: default@cluster_mv_3@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_3 +POSTHOOK: Input: default@cluster_mv_3@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=349.0 +#### A masked pattern was here #### +val_238 238 338.0 +val_238 238 338.0 +val_238 238 338.0 +val_238 238 338.0 +val_238_n 238 338.0 +val_238_n 238 338.0 +val_238_n2 238 338.0 +val_238_n2 238 338.0 +PREHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_4 PARTITIONED ON (partkey) CLUSTERED ON (key,`tes"t`,`te*#"s"t`) AS +SELECT src_txn.key + 100 as partkey, value, key, key+1 as `tes"t`, key+2 as `te*#"s"t` FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@cluster_mv_4 +PREHOOK: Output: default@cluster_mv_4 +POSTHOOK: query: CREATE MATERIALIZED VIEW cluster_mv_4 PARTITIONED ON (partkey) CLUSTERED ON (key,`tes"t`,`te*#"s"t`) AS +SELECT src_txn.key + 100 as partkey, value, key, key+1 as `tes"t`, key+2 as `te*#"s"t` FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cluster_mv_4 +POSTHOOK: Output: default@cluster_mv_4@partkey=301.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=302.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=303.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=305.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=307.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=308.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=309.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=313.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=314.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=316.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=317.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=318.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=319.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=321.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=322.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=323.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=324.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=326.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=328.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=329.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=330.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=333.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=335.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=337.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=338.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=339.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=341.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=342.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=344.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=347.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=348.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=349.0 +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cluster_mv_4 +POSTHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cluster_mv_4 +# col_name data_type comment +value string +key string +tes"t double +te*#"s"t double + +# Partition Information +# col_name data_type comment +partkey double + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + materializedview.distribute.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + materializedview.sort.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + numFiles 32 + numPartitions 32 + numRows 57 + rawDataSize 11062 + totalSize 17102 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT src_txn.key + 100 as partkey, value, key, key+1 as tes"t, key+2 as te*#"s"t FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `value`, `key`, `tes"t`, `te*#"s"t`, `partkey` FROM (SELECT `src_txn`.`key` + 100 as `partkey`, `src_txn`.`value`, `src_txn`.`key`, `src_txn`.`key`+1 as `tes"t`, `src_txn`.`key`+2 as `te*#"s"t` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250) `cluster_mv_4` +Rewrite Enabled: Yes +Outdated for Rewriting: No +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 503 Data size: 91043 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), (UDFToDouble(key) + 100.0D) (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: double) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 5 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col2 (type: double) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string), VALUE._col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + outputColumnNames: value, key, partkey + Statistics: Num rows: 55 Data size: 10395 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + keys: partkey (type: double) + minReductionHashAggr: 0.5090909 + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 23976 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + partkey + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.cluster_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@cluster_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@cluster_mv_3@partkey=301.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=302.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=303.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=305.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=307.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=308.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=309.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=313.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=314.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=316.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=317.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=318.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=319.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=321.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=322.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=323.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=324.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=326.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=328.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=329.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=330.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=333.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=335.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=337.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=338.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=339.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=341.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=342.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=344.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=347.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=348.0 +POSTHOOK: Output: default@cluster_mv_3@partkey=349.0 +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=301.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=301.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=302.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=302.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=303.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=303.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=305.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=305.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=307.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=307.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=308.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=308.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=309.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=309.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=313.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=313.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=314.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=314.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=316.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=316.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=317.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=317.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=318.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=318.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=319.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=319.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=321.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=321.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=322.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=322.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=323.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=323.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=324.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=324.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=326.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=326.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=328.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=328.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=329.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=329.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=330.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=330.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=333.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=333.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=335.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=335.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=337.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=337.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=339.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=339.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=341.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=341.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=342.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=342.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=344.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=344.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=347.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=347.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=348.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=348.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=349.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_3 PARTITION(partkey=349.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_3 +PREHOOK: Input: default@cluster_mv_3@partkey=301.0 +PREHOOK: Input: default@cluster_mv_3@partkey=302.0 +PREHOOK: Input: default@cluster_mv_3@partkey=303.0 +PREHOOK: Input: default@cluster_mv_3@partkey=305.0 +PREHOOK: Input: default@cluster_mv_3@partkey=307.0 +PREHOOK: Input: default@cluster_mv_3@partkey=308.0 +PREHOOK: Input: default@cluster_mv_3@partkey=309.0 +PREHOOK: Input: default@cluster_mv_3@partkey=313.0 +PREHOOK: Input: default@cluster_mv_3@partkey=314.0 +PREHOOK: Input: default@cluster_mv_3@partkey=316.0 +PREHOOK: Input: default@cluster_mv_3@partkey=317.0 +PREHOOK: Input: default@cluster_mv_3@partkey=318.0 +PREHOOK: Input: default@cluster_mv_3@partkey=319.0 +PREHOOK: Input: default@cluster_mv_3@partkey=321.0 +PREHOOK: Input: default@cluster_mv_3@partkey=322.0 +PREHOOK: Input: default@cluster_mv_3@partkey=323.0 +PREHOOK: Input: default@cluster_mv_3@partkey=324.0 +PREHOOK: Input: default@cluster_mv_3@partkey=326.0 +PREHOOK: Input: default@cluster_mv_3@partkey=328.0 +PREHOOK: Input: default@cluster_mv_3@partkey=329.0 +PREHOOK: Input: default@cluster_mv_3@partkey=330.0 +PREHOOK: Input: default@cluster_mv_3@partkey=333.0 +PREHOOK: Input: default@cluster_mv_3@partkey=335.0 +PREHOOK: Input: default@cluster_mv_3@partkey=337.0 +PREHOOK: Input: default@cluster_mv_3@partkey=338.0 +PREHOOK: Input: default@cluster_mv_3@partkey=339.0 +PREHOOK: Input: default@cluster_mv_3@partkey=341.0 +PREHOOK: Input: default@cluster_mv_3@partkey=342.0 +PREHOOK: Input: default@cluster_mv_3@partkey=344.0 +PREHOOK: Input: default@cluster_mv_3@partkey=347.0 +PREHOOK: Input: default@cluster_mv_3@partkey=348.0 +PREHOOK: Input: default@cluster_mv_3@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_3 +POSTHOOK: Input: default@cluster_mv_3@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_3@partkey=349.0 +#### A masked pattern was here #### +val_238 238 338.0 +val_238 238 338.0 +val_238 238 338.0 +val_238 238 338.0 +val_238_n 238 338.0 +val_238_n 238 338.0 +val_238_n2 238 338.0 +val_238_n2 238 338.0 +val_238_n2 238 338.0 +val_238_n2 238 338.0 +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_4 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 503 Data size: 91043 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9955 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string), (UDFToDouble(key) + 1.0D) (type: double), (UDFToDouble(key) + 2.0D) (type: double), (UDFToDouble(key) + 100.0D) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 55 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: double), _col3 (type: double) + sort order: +++ + Map-reduce partition columns: _col1 (type: string), _col2 (type: double), _col3 (type: double) + Statistics: Num rows: 55 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string), _col4 (type: double) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: double), KEY._col3 (type: double), VALUE._col4 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 55 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_4 + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: double), _col4 (type: double) + outputColumnNames: value, key, tes"t, te*#"s"t, partkey + Statistics: Num rows: 55 Data size: 11275 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll'), compute_stats(tes"t, 'hll'), compute_stats(te*#"s"t, 'hll') + keys: partkey (type: double) + minReductionHashAggr: 0.5090909 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 27 Data size: 46872 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + + Map-reduce partition columns: _col0 (type: double) + Statistics: Num rows: 27 Data size: 46872 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: double) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 27 Data size: 47736 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 27 Data size: 47736 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 27 Data size: 47736 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + partkey + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.cluster_mv_4 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key, tes"t, te*#"s"t + Column Types: string, string, double, double + Table: default.cluster_mv_4 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@cluster_mv_4 +POSTHOOK: query: ALTER MATERIALIZED VIEW cluster_mv_4 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@cluster_mv_4@partkey=301.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=302.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=303.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=305.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=307.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=308.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=309.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=313.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=314.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=316.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=317.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=318.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=319.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=321.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=322.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=323.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=324.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=326.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=328.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=329.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=330.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=333.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=335.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=337.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=338.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=339.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=341.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=342.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=344.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=347.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=348.0 +POSTHOOK: Output: default@cluster_mv_4@partkey=349.0 +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=301.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=302.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=303.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=305.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=307.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=308.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=309.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=313.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=314.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=316.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=317.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=318.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=319.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=321.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=322.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=323.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=324.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=326.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=328.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=329.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=330.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=333.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=335.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=337.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=338.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=339.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=341.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=342.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=344.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=347.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=348.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).te*#"s"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).tes"t EXPRESSION [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: cluster_mv_4 PARTITION(partkey=349.0).value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM cluster_mv_4 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@cluster_mv_4 +PREHOOK: Input: default@cluster_mv_4@partkey=301.0 +PREHOOK: Input: default@cluster_mv_4@partkey=302.0 +PREHOOK: Input: default@cluster_mv_4@partkey=303.0 +PREHOOK: Input: default@cluster_mv_4@partkey=305.0 +PREHOOK: Input: default@cluster_mv_4@partkey=307.0 +PREHOOK: Input: default@cluster_mv_4@partkey=308.0 +PREHOOK: Input: default@cluster_mv_4@partkey=309.0 +PREHOOK: Input: default@cluster_mv_4@partkey=313.0 +PREHOOK: Input: default@cluster_mv_4@partkey=314.0 +PREHOOK: Input: default@cluster_mv_4@partkey=316.0 +PREHOOK: Input: default@cluster_mv_4@partkey=317.0 +PREHOOK: Input: default@cluster_mv_4@partkey=318.0 +PREHOOK: Input: default@cluster_mv_4@partkey=319.0 +PREHOOK: Input: default@cluster_mv_4@partkey=321.0 +PREHOOK: Input: default@cluster_mv_4@partkey=322.0 +PREHOOK: Input: default@cluster_mv_4@partkey=323.0 +PREHOOK: Input: default@cluster_mv_4@partkey=324.0 +PREHOOK: Input: default@cluster_mv_4@partkey=326.0 +PREHOOK: Input: default@cluster_mv_4@partkey=328.0 +PREHOOK: Input: default@cluster_mv_4@partkey=329.0 +PREHOOK: Input: default@cluster_mv_4@partkey=330.0 +PREHOOK: Input: default@cluster_mv_4@partkey=333.0 +PREHOOK: Input: default@cluster_mv_4@partkey=335.0 +PREHOOK: Input: default@cluster_mv_4@partkey=337.0 +PREHOOK: Input: default@cluster_mv_4@partkey=338.0 +PREHOOK: Input: default@cluster_mv_4@partkey=339.0 +PREHOOK: Input: default@cluster_mv_4@partkey=341.0 +PREHOOK: Input: default@cluster_mv_4@partkey=342.0 +PREHOOK: Input: default@cluster_mv_4@partkey=344.0 +PREHOOK: Input: default@cluster_mv_4@partkey=347.0 +PREHOOK: Input: default@cluster_mv_4@partkey=348.0 +PREHOOK: Input: default@cluster_mv_4@partkey=349.0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM cluster_mv_4 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cluster_mv_4 +POSTHOOK: Input: default@cluster_mv_4@partkey=301.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=302.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=303.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=305.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=307.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=308.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=309.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=313.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=314.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=316.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=317.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=318.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=319.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=321.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=322.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=323.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=324.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=326.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=328.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=329.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=330.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=333.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=335.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=337.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=338.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=339.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=341.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=342.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=344.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=347.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=348.0 +POSTHOOK: Input: default@cluster_mv_4@partkey=349.0 +#### A masked pattern was here #### +val_238 238 239.0 240.0 338.0 +val_238 238 239.0 240.0 338.0 +val_238_n 238 239.0 240.0 338.0 +val_238_n2 238 239.0 240.0 338.0 +val_238_n2 238 239.0 240.0 338.0 +PREHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@cluster_mv_4 +POSTHOOK: query: DESCRIBE FORMATTED cluster_mv_4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@cluster_mv_4 +# col_name data_type comment +value string +key string +tes"t double +te*#"s"t double + +# Partition Information +# col_name data_type comment +partkey double + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 + materializedview.distribute.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + materializedview.sort.columns [\"key\",\"tes\\\"t\",\"te*#\\\"s\\\"t\"] + numFiles 32 + numPartitions 32 + numRows 58 + rawDataSize 11257 + totalSize 17105 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT src_txn.key + 100 as partkey, value, key, key+1 as tes"t, key+2 as te*#"s"t FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `value`, `key`, `tes"t`, `te*#"s"t`, `partkey` FROM (SELECT `src_txn`.`key` + 100 as `partkey`, `src_txn`.`value`, `src_txn`.`key`, `src_txn`.`key`+1 as `tes"t`, `src_txn`.`key`+2 as `te*#"s"t` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250) `cluster_mv_4` +Rewrite Enabled: Yes +Outdated for Rewriting: No diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_sort.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_sort.q.out new file mode 100644 index 0000000000..d610468064 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/materialized_view_sort.q.out @@ -0,0 +1,909 @@ +PREHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn +POSTHOOK: query: CREATE TABLE src_txn stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW sort_mv_1 SORTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@sort_mv_1 +POSTHOOK: query: EXPLAIN +CREATE MATERIALIZED VIEW sort_mv_1 SORTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sort_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-0, Stage-2 + Stage-3 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.sort_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: col1, col2 + Statistics: Num rows: 55 Data size: 9790 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(col1, 'hll'), compute_stats(col2, 'hll') + minReductionHashAggr: 0.9818182 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-4 + Create View + columns: value string, key string + sort columns: key string + expanded text: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 + name: default.sort_mv_1 + original text: SELECT value, key FROM src_txn where key > 200 and key < 250 + rewrite enabled: true + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.sort_mv_1 + + Stage: Stage-5 + Materialized View Work + + Stage: Stage-0 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: CREATE MATERIALIZED VIEW sort_mv_1 SORTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@sort_mv_1 +POSTHOOK: query: CREATE MATERIALIZED VIEW sort_mv_1 SORTED ON (key) STORED AS TEXTFILE AS +SELECT value, key FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sort_mv_1 +PREHOOK: query: DESCRIBE FORMATTED sort_mv_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sort_mv_1 +POSTHOOK: query: DESCRIBE FORMATTED sort_mv_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sort_mv_1 +# col_name data_type comment +value string +key string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MATERIALIZED_VIEW +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 + materializedview.sort.columns [\"key\"] + numFiles 1 + numRows 55 + rawDataSize 605 + totalSize 660 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# Materialized View Information +Original Query: SELECT value, key FROM src_txn where key > 200 and key < 250 +Expanded Query: SELECT `src_txn`.`value`, `src_txn`.`key` FROM `default`.`src_txn` where `src_txn`.`key` > 200 and `src_txn`.`key` < 250 +Rewrite Enabled: Yes +Outdated for Rewriting: No +Found 1 items +#### A masked pattern was here #### +val_201201 +val_202202 +val_203203 +val_203203 +val_205205 +val_205205 +val_207207 +val_207207 +val_208208 +val_208208 +val_208208 +val_209209 +val_209209 +val_213213 +val_213213 +val_214214 +val_216216 +val_216216 +val_217217 +val_217217 +val_218218 +val_219219 +val_219219 +val_221221 +val_221221 +val_222222 +val_223223 +val_223223 +val_224224 +val_224224 +val_226226 +val_228228 +val_229229 +val_229229 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_233233 +val_233233 +val_235235 +val_237237 +val_237237 +val_238238 +val_238238 +val_239239 +val_239239 +val_241241 +val_242242 +val_242242 +val_244244 +val_247247 +val_248248 +val_249249 +PREHOOK: query: EXPLAIN +SELECT * FROM sort_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM sort_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: sort_mv_1 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM sort_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM sort_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_1 +#### A masked pattern was here #### +val_238 238 +val_238 238 +PREHOOK: query: CREATE MATERIALIZED VIEW sort_mv_2 SORTED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Output: database:default +PREHOOK: Output: default@sort_mv_2 +POSTHOOK: query: CREATE MATERIALIZED VIEW sort_mv_2 SORTED ON (value) STORED AS TEXTFILE AS +SELECT key, value FROM src_txn where key > 200 and key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sort_mv_2 +Found 1 items +#### A masked pattern was here #### +201val_201 +202val_202 +203val_203 +203val_203 +205val_205 +205val_205 +207val_207 +207val_207 +208val_208 +208val_208 +208val_208 +209val_209 +209val_209 +213val_213 +213val_213 +214val_214 +216val_216 +216val_216 +217val_217 +217val_217 +218val_218 +219val_219 +219val_219 +221val_221 +221val_221 +222val_222 +223val_223 +223val_223 +224val_224 +224val_224 +226val_226 +228val_228 +229val_229 +229val_229 +230val_230 +230val_230 +230val_230 +230val_230 +230val_230 +233val_233 +233val_233 +235val_235 +237val_237 +237val_237 +238val_238 +238val_238 +239val_239 +239val_239 +241val_241 +242val_242 +242val_242 +244val_244 +247val_247 +248val_248 +249val_249 +PREHOOK: query: EXPLAIN +SELECT * FROM sort_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM sort_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: sort_mv_2 + filterExpr: (value = 'val_238') (type: boolean) + Filter Operator + predicate: (value = 'val_238') (type: boolean) + Select Operator + expressions: key (type: string), 'val_238' (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * FROM sort_mv_2 where value = 'val_238' +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM sort_mv_2 where value = 'val_238' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: EXPLAIN +SELECT value FROM sort_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT value FROM sort_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: sort_mv_2 + filterExpr: (UDFToDouble(key) = 238.0D) (type: boolean) + Filter Operator + predicate: (UDFToDouble(key) = 238.0D) (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: SELECT value FROM sort_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM sort_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +val_238 +val_238 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW sort_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@sort_mv_1 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW sort_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@sort_mv_1 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 501 Data size: 90180 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 1L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.sort_mv_1 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 18 Data size: 3240 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9444444 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Execution mode: llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.sort_mv_1 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.sort_mv_1 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW sort_mv_1 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Output: default@sort_mv_1 +POSTHOOK: query: ALTER MATERIALIZED VIEW sort_mv_1 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Output: default@sort_mv_1 +POSTHOOK: Lineage: sort_mv_1.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: sort_mv_1.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM sort_mv_1 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM sort_mv_1 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_1 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238_n 238 +PREHOOK: query: SELECT * FROM sort_mv_2 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM sort_mv_2 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_2 +#### A masked pattern was here #### +238 val_238 +238 val_238 +PREHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_txn_2 +POSTHOOK: query: CREATE TABLE src_txn_2 stored as orc TBLPROPERTIES ('transactional' = 'true') +AS SELECT * FROM src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_txn_2 +POSTHOOK: Lineage: src_txn_2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_txn_2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: CREATE MATERIALIZED VIEW sort_mv_3 SORTED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: database:default +PREHOOK: Output: default@sort_mv_3 +POSTHOOK: query: CREATE MATERIALIZED VIEW sort_mv_3 SORTED ON (key) STORED AS TEXTFILE AS +SELECT src_txn.value, src_txn.key FROM src_txn, src_txn_2 +WHERE src_txn.key = src_txn_2.key + AND src_txn.key > 200 AND src_txn.key < 250 +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sort_mv_3 +Found 1 items +#### A masked pattern was here #### +val_201201 +val_202202 +val_203203 +val_203203 +val_203203 +val_203203 +val_205205 +val_205205 +val_205205 +val_205205 +val_207207 +val_207207 +val_207207 +val_207207 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_208208 +val_209209 +val_209209 +val_209209 +val_209209 +val_213213 +val_213213 +val_213213 +val_213213 +val_214214 +val_216216 +val_216216 +val_216216 +val_216216 +val_217217 +val_217217 +val_217217 +val_217217 +val_218218 +val_219219 +val_219219 +val_219219 +val_219219 +val_221221 +val_221221 +val_221221 +val_221221 +val_222222 +val_223223 +val_223223 +val_223223 +val_223223 +val_224224 +val_224224 +val_224224 +val_224224 +val_226226 +val_228228 +val_229229 +val_229229 +val_229229 +val_229229 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_230230 +val_233233 +val_233233 +val_233233 +val_233233 +val_235235 +val_237237 +val_237237 +val_237237 +val_237237 +val_238_n238 +val_238238 +val_238238 +val_238238 +val_238238 +val_238_n238 +val_239239 +val_239239 +val_239239 +val_239239 +val_241241 +val_242242 +val_242242 +val_242242 +val_242242 +val_244244 +val_247247 +val_248248 +val_249249 +PREHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@src_txn +POSTHOOK: query: INSERT INTO src_txn VALUES (238, 'val_238_n2') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@src_txn +POSTHOOK: Lineage: src_txn.key SCRIPT [] +POSTHOOK: Lineage: src_txn.value SCRIPT [] +PREHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW sort_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@sort_mv_3 +POSTHOOK: query: EXPLAIN +ALTER MATERIALIZED VIEW sort_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@sort_mv_3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_txn + filterExpr: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 502 Data size: 90862 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((ROW__ID.writeid > 2L) and (UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Map 4 + Map Operator Tree: + TableScan + alias: src_txn_2 + filterExpr: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((UDFToDouble(key) > 200.0D) and (UDFToDouble(key) < 250.0D)) (type: boolean) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 55 Data size: 4785 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: may be used (ACID table) + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.sort_mv_3 + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: value, key + Statistics: Num rows: 18 Data size: 3258 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(value, 'hll'), compute_stats(key, 'hll') + minReductionHashAggr: 0.9444444 + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: struct), _col1 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 880 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.sort_mv_3 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: value, key + Column Types: string, string + Table: default.sort_mv_3 + + Stage: Stage-4 + Materialized View Work + +PREHOOK: query: ALTER MATERIALIZED VIEW sort_mv_3 REBUILD +PREHOOK: type: QUERY +PREHOOK: Input: default@src_txn +PREHOOK: Input: default@src_txn_2 +PREHOOK: Output: default@sort_mv_3 +POSTHOOK: query: ALTER MATERIALIZED VIEW sort_mv_3 REBUILD +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_txn +POSTHOOK: Input: default@src_txn_2 +POSTHOOK: Output: default@sort_mv_3 +POSTHOOK: Lineage: sort_mv_3.key SIMPLE [(src_txn)src_txn.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: sort_mv_3.value SIMPLE [(src_txn)src_txn.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM sort_mv_3 where key = 238 +PREHOOK: type: QUERY +PREHOOK: Input: default@sort_mv_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM sort_mv_3 where key = 238 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sort_mv_3 +#### A masked pattern was here #### +val_238 238 +val_238 238 +val_238 238 +val_238 238 +val_238_n 238 +val_238_n 238 +val_238_n2 238 +val_238_n2 238