From 474fa12194508943a574c226ee3327c65980e150 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Fri, 22 May 2015 18:26:52 -0700 Subject: [PATCH] HIVE-10807 : Invalidate basic stats for insert queries if autogather=false --- .../org/apache/hadoop/hive/ql/QueryProperties.java | 10 -- .../hadoop/hive/ql/optimizer/GenMRFileSink1.java | 1 + .../hadoop/hive/ql/optimizer/GenMapRedUtils.java | 66 ++++----- .../apache/hadoop/hive/ql/parse/QBParseInfo.java | 9 -- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 10 +- ql/src/test/queries/clientpositive/insert_into1.q | 12 +- .../stats_partialscan_autogether.q.out | 6 +- .../test/results/clientpositive/insert_into1.q.out | 155 +++++++++++++++++++++ .../clientpositive/skewjoin_union_remove_1.q.out | 20 ++- .../clientpositive/spark/insert_into1.q.out | 120 ++++++++++++++++ .../spark/skewjoin_union_remove_1.q.out | 8 ++ .../results/clientpositive/spark/stats12.q.out | 24 ++-- .../results/clientpositive/spark/stats13.q.out | 24 ++-- .../test/results/clientpositive/spark/stats2.q.out | 4 + .../test/results/clientpositive/spark/stats5.q.out | 2 +- .../test/results/clientpositive/spark/stats6.q.out | 12 +- .../test/results/clientpositive/spark/stats9.q.out | 2 +- .../clientpositive/spark/stats_noscan_1.q.out | 40 +++--- .../clientpositive/spark/stats_partscan_1_23.q.out | 18 +-- .../clientpositive/spark/union_remove_1.q.out | 10 +- .../clientpositive/spark/union_remove_10.q.out | 28 ++-- .../clientpositive/spark/union_remove_11.q.out | 28 ++-- .../clientpositive/spark/union_remove_12.q.out | 34 +++-- .../clientpositive/spark/union_remove_13.q.out | 34 +++-- .../clientpositive/spark/union_remove_14.q.out | 34 +++-- .../clientpositive/spark/union_remove_15.q.out | 4 + .../clientpositive/spark/union_remove_16.q.out | 22 +-- .../clientpositive/spark/union_remove_17.q.out | 4 + .../clientpositive/spark/union_remove_18.q.out | 4 + .../clientpositive/spark/union_remove_19.q.out | 18 ++- .../clientpositive/spark/union_remove_2.q.out | 10 +- .../clientpositive/spark/union_remove_20.q.out | 10 +- .../clientpositive/spark/union_remove_21.q.out | 10 +- .../clientpositive/spark/union_remove_22.q.out | 14 +- .../clientpositive/spark/union_remove_23.q.out | 10 +- .../clientpositive/spark/union_remove_24.q.out | 10 +- .../clientpositive/spark/union_remove_25.q.out | 30 ++-- .../clientpositive/spark/union_remove_3.q.out | 10 +- .../clientpositive/spark/union_remove_4.q.out | 28 ++-- .../clientpositive/spark/union_remove_5.q.out | 28 ++-- .../clientpositive/spark/union_remove_6.q.out | 8 ++ .../clientpositive/spark/union_remove_6_subq.q.out | 8 ++ .../clientpositive/spark/union_remove_7.q.out | 10 +- .../clientpositive/spark/union_remove_8.q.out | 10 +- .../clientpositive/spark/union_remove_9.q.out | 28 ++-- ql/src/test/results/clientpositive/stats12.q.out | 24 ++-- ql/src/test/results/clientpositive/stats13.q.out | 24 ++-- ql/src/test/results/clientpositive/stats2.q.out | 4 + ql/src/test/results/clientpositive/stats5.q.out | 2 +- ql/src/test/results/clientpositive/stats6.q.out | 12 +- ql/src/test/results/clientpositive/stats9.q.out | 2 +- .../results/clientpositive/stats_noscan_1.q.out | 40 +++--- .../clientpositive/stats_partscan_1_23.q.out | 12 +- .../results/clientpositive/tez/insert_into1.q.out | 124 +++++++++++++++++ .../clientpositive/tez/stats_noscan_1.q.out | 40 +++--- .../results/clientpositive/union_remove_1.q.out | 14 +- .../results/clientpositive/union_remove_10.q.out | 34 +++-- .../results/clientpositive/union_remove_11.q.out | 28 ++-- .../results/clientpositive/union_remove_12.q.out | 36 ++--- .../results/clientpositive/union_remove_13.q.out | 36 ++--- .../results/clientpositive/union_remove_14.q.out | 36 ++--- .../results/clientpositive/union_remove_15.q.out | 8 +- .../results/clientpositive/union_remove_16.q.out | 26 ++-- .../results/clientpositive/union_remove_17.q.out | 4 + .../results/clientpositive/union_remove_18.q.out | 8 +- .../results/clientpositive/union_remove_19.q.out | 30 ++-- .../results/clientpositive/union_remove_2.q.out | 16 ++- .../results/clientpositive/union_remove_20.q.out | 14 +- .../results/clientpositive/union_remove_21.q.out | 14 +- .../results/clientpositive/union_remove_22.q.out | 22 ++- .../results/clientpositive/union_remove_23.q.out | 16 ++- .../results/clientpositive/union_remove_24.q.out | 14 +- .../results/clientpositive/union_remove_25.q.out | 42 ++++-- .../results/clientpositive/union_remove_3.q.out | 10 +- .../results/clientpositive/union_remove_4.q.out | 32 +++-- .../results/clientpositive/union_remove_5.q.out | 34 +++-- .../results/clientpositive/union_remove_6.q.out | 14 +- .../clientpositive/union_remove_6_subq.q.out | 14 +- .../results/clientpositive/union_remove_7.q.out | 14 +- .../results/clientpositive/union_remove_8.q.out | 16 ++- .../results/clientpositive/union_remove_9.q.out | 32 +++-- 81 files changed, 1256 insertions(+), 579 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index e8f7fba..3bc9432 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -39,7 +39,6 @@ boolean noScanAnalyzeCommand; boolean analyzeRewrite; boolean ctas; - boolean insertToTable; int outerQueryLimit; boolean hasJoin = false; @@ -115,14 +114,6 @@ public void setCTAS(boolean ctas) { this.ctas = ctas; } - public boolean isInsertToTable() { - return insertToTable; - } - - public void setInsertToTable(boolean insertToTable) { - this.insertToTable = insertToTable; - } - public int getOuterQueryLimit() { return outerQueryLimit; } @@ -276,7 +267,6 @@ public void clear() { noScanAnalyzeCommand = false; analyzeRewrite = false; ctas = false; - insertToTable = false; outerQueryLimit = -1; hasJoin = false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java index e5b9c2b..588909b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java @@ -61,6 +61,7 @@ public GenMRFileSink1() { * @param opProcCtx * context */ + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException { GenMRProcContext ctx = (GenMRProcContext) opProcCtx; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index acd9bf5..477a90f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -983,7 +983,7 @@ public static TableScanOperator createTemporaryFile( fileSinkOp.setParentOperators(Utilities.makeList(parent)); // Create a dummy TableScanOperator for the file generated through fileSinkOp - TableScanOperator tableScanOp = (TableScanOperator) createTemporaryTableScanOperator( + TableScanOperator tableScanOp = createTemporaryTableScanOperator( parent.getSchema()); // Connect this TableScanOperator to child. @@ -1419,37 +1419,42 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, assert statsWork != null : "Error when genereting StatsTask"; statsWork.setSourceTask(currTask); - statsWork.setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); - - if (currTask.getWork() instanceof MapredWork) { - MapredWork mrWork = (MapredWork) currTask.getWork(); - mrWork.getMapWork().setGatheringStats(true); - if (mrWork.getReduceWork() != null) { - mrWork.getReduceWork().setGatheringStats(true); - } - } else if (currTask.getWork() instanceof SparkWork) { - SparkWork work = (SparkWork) currTask.getWork(); - for (BaseWork w: work.getAllWork()) { - w.setGatheringStats(true); - } - } else { // must be TezWork - TezWork work = (TezWork) currTask.getWork(); - for (BaseWork w: work.getAllWork()) { - w.setGatheringStats(true); + if (hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)) { + statsWork.setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); + + if (currTask.getWork() instanceof MapredWork) { + MapredWork mrWork = (MapredWork) currTask.getWork(); + mrWork.getMapWork().setGatheringStats(true); + if (mrWork.getReduceWork() != null) { + mrWork.getReduceWork().setGatheringStats(true); + } + } else if (currTask.getWork() instanceof SparkWork) { + SparkWork work = (SparkWork) currTask.getWork(); + for (BaseWork w: work.getAllWork()) { + w.setGatheringStats(true); + } + } else { // must be TezWork + TezWork work = (TezWork) currTask.getWork(); + for (BaseWork w: work.getAllWork()) { + w.setGatheringStats(true); + } } - } - // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix - // in FileSinkDesc is used for stats publishing. They should be consistent. - statsWork.setAggKey(nd.getConf().getStatsAggPrefix()); - Task statsTask = TaskFactory.get(statsWork, hconf); + // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix + // in FileSinkDesc is used for stats publishing. They should be consistent. + statsWork.setAggKey(nd.getConf().getStatsAggPrefix()); - // mark the MapredWork and FileSinkOperator for gathering stats - nd.getConf().setGatherStats(true); - nd.getConf().setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); - nd.getConf().setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(hconf)); - // mrWork.addDestinationTable(nd.getConf().getTableInfo().getTableName()); + // mark the MapredWork and FileSinkOperator for gathering stats + nd.getConf().setGatherStats(true); + nd.getConf().setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); + nd.getConf().setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(hconf)); + } else { + statsWork.setClearAggregatorStats(true); + statsWork.setNoStatsAggregator(true); + statsWork.setStatsReliable(false); + } + Task statsTask = TaskFactory.get(statsWork, hconf); // subscribe feeds from the MoveTask so that MoveTask can forward the list // of dynamic partition list to the StatsTask mvTask.addDependentTask(statsTask); @@ -1464,8 +1469,7 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, * @return */ public static boolean isInsertInto(ParseContext parseCtx, FileSinkOperator fsOp) { - return fsOp.getConf().getTableInfo().getTableName() != null && - parseCtx.getQueryProperties().isInsertToTable(); + return fsOp.getConf().getTableInfo().getTableName() != null; } /** @@ -1687,7 +1691,7 @@ public static boolean isMergeRequired(List> mvTasks, HiveConf hco // no need of merging if the move is to a local file system MoveTask mvTask = (MoveTask) GenMapRedUtils.findMoveTask(mvTasks, fsOp); - if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)) { + if (mvTask != null && isInsertTable) { GenMapRedUtils.addStatsTask(fsOp, mvTask, currTask, hconf); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 14a7e9c..9072d7f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -65,7 +65,6 @@ private final HashSet insertIntoTables; private boolean isAnalyzeCommand; // used for the analyze command (statistics) - private boolean isInsertToTable; // used for insert overwrite command (statistics) private boolean isNoScanAnalyzeCommand; // used for the analyze command (statistics) (noscan) private boolean isPartialScanAnalyzeCommand; // used for the analyze command (statistics) // (partialscan) @@ -550,14 +549,6 @@ public boolean isAnalyzeCommand() { return isAnalyzeCommand; } - public void setIsInsertToTable(boolean isInsertToTable) { - this.isInsertToTable = isInsertToTable; - } - - public boolean isInsertToTable() { - return isInsertToTable; - } - public void addTableSpec(String tName, TableSpec tSpec) { tableSpecs.put(tName, tSpec); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 351c267..c445533 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -730,7 +730,7 @@ private ASTNode genValuesTempTable(ASTNode originalFrom, QB qb) throws SemanticE Path dataDir = null; if(!qb.getEncryptedTargetTablePaths().isEmpty()) { //currently only Insert into T values(...) is supported thus only 1 values clause - //and only 1 target table are possible. If/when support for + //and only 1 target table are possible. If/when support for //select ... from values(...) is added an insert statement may have multiple //encrypted target tables. dataDir = ctx.getMRTmpPath(qb.getEncryptedTargetTablePaths().get(0).toUri()); @@ -1761,8 +1761,6 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException qb.getMetaData().setDestForAlias(name, ts.partHandle); } if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - // Set that variable to automatically collect stats during the MapReduce job - qb.getParseInfo().setIsInsertToTable(true); // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } @@ -1799,8 +1797,6 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException } if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { TableSpec ts = new TableSpec(db, conf, this.ast); - // Set that variable to automatically collect stats during the MapReduce job - qb.getParseInfo().setIsInsertToTable(true); // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } @@ -6379,7 +6375,6 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // verify that our destination is empty before proceeding if (dest_tab.isImmutable() && qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),dest_tab.getTableName())){ - qb.getParseInfo().isInsertToTable(); try { FileSystem fs = partPath.getFileSystem(conf); if (! MetaStoreUtils.isDirEmpty(fs,partPath)){ @@ -10070,7 +10065,7 @@ private void preProcessForInsert(ASTNode node, QB qb) throws SemanticException { return; } for (Node child : node.getChildren()) { - //each insert of multi insert looks like + //each insert of multi insert looks like //(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME T1))) if (((ASTNode) child).getToken().getType() != HiveParser.TOK_INSERT) { continue; @@ -12199,7 +12194,6 @@ private void copyInfoToQueryProperties(QueryProperties queryProperties) { queryProperties.setNoScanAnalyzeCommand(qb.getParseInfo().isNoScanAnalyzeCommand()); queryProperties.setAnalyzeRewrite(qb.isAnalyzeRewrite()); queryProperties.setCTAS(qb.getTableDesc() != null); - queryProperties.setInsertToTable(qb.getParseInfo().isInsertToTable()); queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() && !qb.getParseInfo().getDestToOrderBy().isEmpty()); queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit()); diff --git a/ql/src/test/queries/clientpositive/insert_into1.q b/ql/src/test/queries/clientpositive/insert_into1.q index f19506a..311963d 100644 --- a/ql/src/test/queries/clientpositive/insert_into1.q +++ b/ql/src/test/queries/clientpositive/insert_into1.q @@ -31,6 +31,16 @@ explain SELECT COUNT(*) FROM insert_into1; select count(*) from insert_into1; -DROP TABLE insert_into1; +set hive.stats.autogather=false; +explain +insert into table insert_into1 values(1, 'abc'); +insert into table insert_into1 values(1, 'abc'); +explain +SELECT COUNT(*) FROM insert_into1; +select count(*) from insert_into1; + + +DROP TABLE insert_into1; +set hive.stats.autogather=true; set hive.compute.query.using.stats=false; diff --git a/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out b/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out index 321ebe5..be15852 100644 --- a/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out +++ b/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out @@ -68,10 +68,10 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5077 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/insert_into1.q.out b/ql/src/test/results/clientpositive/insert_into1.q.out index 9e5f3bb..48194dc 100644 --- a/ql/src/test/results/clientpositive/insert_into1.q.out +++ b/ql/src/test/results/clientpositive/insert_into1.q.out @@ -337,6 +337,161 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@insert_into1 #### A masked pattern was here #### 10 +PREHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: insert_into1 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from insert_into1 +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from insert_into1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +11 PREHOOK: query: DROP TABLE insert_into1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@insert_into1 diff --git a/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out b/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out index 1f21877..9ddccaf 100644 --- a/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out @@ -336,8 +336,9 @@ SELECT * FROM T1 a JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage + Stage-0 depends on stages: Stage-1, Stage-4 + Stage-2 depends on stages: Stage-0 + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -399,7 +400,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-3 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -487,8 +491,9 @@ SELECT * FROM T1 a RIGHT OUTER JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage + Stage-0 depends on stages: Stage-1, Stage-4 + Stage-2 depends on stages: Stage-0 + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -550,7 +555,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-3 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/spark/insert_into1.q.out b/ql/src/test/results/clientpositive/spark/insert_into1.q.out index e7f7d8f..ee5b271 100644 --- a/ql/src/test/results/clientpositive/spark/insert_into1.q.out +++ b/ql/src/test/results/clientpositive/spark/insert_into1.q.out @@ -355,6 +355,126 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@insert_into1 #### A masked pattern was here #### 10 +PREHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into1 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from insert_into1 +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from insert_into1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +11 PREHOOK: query: DROP TABLE insert_into1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@insert_into1 diff --git a/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out b/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out index f6acc34..564c81d 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoin_union_remove_1.q.out @@ -349,6 +349,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -463,6 +464,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: INSERT OVERWRITE TABLE DEST1 SELECT * FROM T1 a JOIN T2 b ON a.key = b.key PREHOOK: type: QUERY @@ -506,6 +510,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -620,6 +625,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: INSERT OVERWRITE TABLE DEST1 SELECT * FROM T1 a RIGHT OUTER JOIN T2 b ON a.key = b.key PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/stats12.q.out b/ql/src/test/results/clientpositive/spark/stats12.q.out index db575df..09ac780 100644 --- a/ql/src/test/results/clientpositive/spark/stats12.q.out +++ b/ql/src/test/results/clientpositive/spark/stats12.q.out @@ -81,7 +81,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -89,10 +89,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.analyze_srcpart numFiles 1 - numRows -1 + numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize -1 + rawDataSize 0 serialization.ddl struct analyze_srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -127,7 +127,7 @@ STAGE PLANS: ds 2008-04-08 hr 12 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -135,10 +135,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.analyze_srcpart numFiles 1 - numRows -1 + numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize -1 + rawDataSize 0 serialization.ddl struct analyze_srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -334,10 +334,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -376,10 +376,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/stats13.q.out b/ql/src/test/results/clientpositive/spark/stats13.q.out index f38f876..28db710 100644 --- a/ql/src/test/results/clientpositive/spark/stats13.q.out +++ b/ql/src/test/results/clientpositive/spark/stats13.q.out @@ -82,7 +82,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -90,10 +90,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.analyze_srcpart numFiles 1 - numRows -1 + numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize -1 + rawDataSize 0 serialization.ddl struct analyze_srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -242,10 +242,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -284,10 +284,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -326,10 +326,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/stats2.q.out b/ql/src/test/results/clientpositive/spark/stats2.q.out index ddc8226..21e9970 100644 --- a/ql/src/test/results/clientpositive/spark/stats2.q.out +++ b/ql/src/test/results/clientpositive/spark/stats2.q.out @@ -15,6 +15,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -52,6 +53,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.analyze_t1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/spark/stats5.q.out b/ql/src/test/results/clientpositive/spark/stats5.q.out index b61101d..524621e 100644 --- a/ql/src/test/results/clientpositive/spark/stats5.q.out +++ b/ql/src/test/results/clientpositive/spark/stats5.q.out @@ -25,7 +25,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_src - Statistics: Num rows: -1 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 Stats-Aggr Operator diff --git a/ql/src/test/results/clientpositive/spark/stats6.q.out b/ql/src/test/results/clientpositive/spark/stats6.q.out index b4435f2..67e9732 100644 --- a/ql/src/test/results/clientpositive/spark/stats6.q.out +++ b/ql/src/test/results/clientpositive/spark/stats6.q.out @@ -166,10 +166,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -208,10 +208,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/stats9.q.out b/ql/src/test/results/clientpositive/spark/stats9.q.out index 2c7daea..903b5ab 100644 --- a/ql/src/test/results/clientpositive/spark/stats9.q.out +++ b/ql/src/test/results/clientpositive/spark/stats9.q.out @@ -33,7 +33,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcbucket - Statistics: Num rows: -1 Data size: 11603 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 11603 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 Stats-Aggr Operator diff --git a/ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out b/ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out index 80c3092..9961d25 100644 --- a/ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out +++ b/ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out @@ -105,8 +105,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -147,8 +147,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -187,10 +187,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -229,10 +229,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -386,8 +386,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -428,8 +428,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -468,10 +468,10 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -510,10 +510,10 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out b/ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out index adcf150..4ace45b 100644 --- a/ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out @@ -78,10 +78,10 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5293 #### A masked pattern was here #### @@ -153,10 +153,10 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5293 #### A masked pattern was here #### @@ -195,10 +195,10 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5293 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_1.q.out b/ql/src/test/results/clientpositive/spark/union_remove_1.q.out index bf0fc20..0b6c020 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_1.q.out @@ -63,6 +63,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -157,6 +158,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -198,10 +202,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out index a9d4b51..d7785f3 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_10.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_10.q.out @@ -78,12 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -164,10 +165,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -184,6 +185,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -194,7 +198,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -205,7 +209,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -260,10 +264,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 350 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_11.q.out b/ql/src/test/results/clientpositive/spark/union_remove_11.q.out index be65741..a18e7cf 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_11.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_11.q.out @@ -78,12 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -154,10 +155,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -174,6 +175,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -184,7 +188,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -195,7 +199,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -250,10 +254,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 273 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_12.q.out b/ql/src/test/results/clientpositive/spark/union_remove_12.q.out index 62d9d7d..6081720 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_12.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_12.q.out @@ -69,17 +69,18 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key )c POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-8 is a root stage + Stage-1 depends on stages: Stage-8 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-8 Spark #### A masked pattern was here #### Vertices: @@ -160,10 +161,10 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -180,6 +181,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -190,7 +194,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -201,7 +205,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -250,10 +254,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 194 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_13.q.out b/ql/src/test/results/clientpositive/spark/union_remove_13.q.out index 995d180..8386836 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_13.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_13.q.out @@ -69,17 +69,18 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key )c POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-8 is a root stage + Stage-1 depends on stages: Stage-8 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-8 Spark #### A masked pattern was here #### Vertices: @@ -186,10 +187,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -206,6 +207,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -216,7 +220,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -227,7 +231,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -276,10 +280,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 271 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_14.q.out b/ql/src/test/results/clientpositive/spark/union_remove_14.q.out index b28bb75..ca1c2f7 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_14.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_14.q.out @@ -71,17 +71,18 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key )c POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-7 is a root stage - Stage-1 depends on stages: Stage-7 - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-8 is a root stage + Stage-1 depends on stages: Stage-8 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: - Stage: Stage-7 + Stage: Stage-8 Spark #### A masked pattern was here #### Vertices: @@ -162,10 +163,10 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -182,6 +183,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -192,7 +196,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -203,7 +207,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -252,10 +256,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 194 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_15.q.out b/ql/src/test/results/clientpositive/spark/union_remove_15.q.out index 58abd21..adf0df0 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_15.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_15.q.out @@ -69,6 +69,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -173,6 +174,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_16.q.out b/ql/src/test/results/clientpositive/spark/union_remove_16.q.out index 356e79a..7fa371c 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_16.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_16.q.out @@ -66,12 +66,13 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -164,10 +165,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -186,6 +187,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -196,7 +200,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -207,7 +211,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true diff --git a/ql/src/test/results/clientpositive/spark/union_remove_17.q.out b/ql/src/test/results/clientpositive/spark/union_remove_17.q.out index 74a5b23..1dfe70c 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_17.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_17.q.out @@ -63,6 +63,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -124,6 +125,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_18.q.out b/ql/src/test/results/clientpositive/spark/union_remove_18.q.out index 2c01a5c..306fbc7 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_18.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_18.q.out @@ -67,6 +67,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -171,6 +172,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out index d1e2312..deaf4f7 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out @@ -67,6 +67,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -161,6 +162,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, a.`values` FROM ( @@ -202,10 +206,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### @@ -260,6 +264,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -376,6 +381,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, a.`values` FROM ( @@ -437,6 +445,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -545,6 +554,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 select key, `values` from ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_2.q.out b/ql/src/test/results/clientpositive/spark/union_remove_2.q.out index 59d88cb..4c12ec8 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_2.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_2.q.out @@ -69,6 +69,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -159,6 +160,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -204,10 +208,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 68 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_20.q.out b/ql/src/test/results/clientpositive/spark/union_remove_20.q.out index f57353f..1c2c640 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_20.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_20.q.out @@ -65,6 +65,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -167,6 +168,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT a.`values`, a.key FROM ( @@ -208,10 +212,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_21.q.out b/ql/src/test/results/clientpositive/spark/union_remove_21.q.out index 48867fb..a25df40 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_21.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_21.q.out @@ -65,6 +65,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -175,6 +176,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key FROM ( @@ -214,10 +218,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 20 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_22.q.out b/ql/src/test/results/clientpositive/spark/union_remove_22.q.out index c41e12f..a2acb7d 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_22.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_22.q.out @@ -67,6 +67,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -169,6 +170,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, a.`values`, a.`values` FROM ( @@ -212,10 +216,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 60 #### A masked pattern was here #### @@ -268,6 +272,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -370,6 +375,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT a.key, concat(a.`values`, a.`values`), concat(a.`values`, a.`values`) FROM ( diff --git a/ql/src/test/results/clientpositive/spark/union_remove_23.q.out b/ql/src/test/results/clientpositive/spark/union_remove_23.q.out index d9875f3..2b5fc63 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_23.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_23.q.out @@ -67,6 +67,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -189,6 +190,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -232,10 +236,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_24.q.out b/ql/src/test/results/clientpositive/spark/union_remove_24.q.out index e989ed2..616db7b 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_24.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_24.q.out @@ -61,6 +61,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -163,6 +164,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 SELECT * FROM ( @@ -204,10 +208,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 60 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out index a5a0126..64e142f 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out @@ -79,6 +79,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -175,6 +176,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 partition(ds='2004') SELECT * FROM ( @@ -221,10 +225,10 @@ Table: outputtbl1 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### @@ -279,6 +283,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -375,6 +380,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl2 partition(ds) SELECT * FROM ( @@ -432,10 +440,10 @@ Table: outputtbl2 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 6826 #### A masked pattern was here #### @@ -468,6 +476,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -565,6 +574,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl3 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl3 partition(ds, hr) SELECT * FROM ( @@ -627,10 +639,10 @@ Table: outputtbl3 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 6812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_3.q.out b/ql/src/test/results/clientpositive/spark/union_remove_3.q.out index a14a8b4..795d71d 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_3.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_3.q.out @@ -69,6 +69,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -149,6 +150,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -194,10 +198,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 72 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_4.q.out b/ql/src/test/results/clientpositive/spark/union_remove_4.q.out index 7d94d1d..27842c9 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_4.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_4.q.out @@ -62,12 +62,13 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -152,10 +153,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -172,6 +173,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -186,7 +190,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -201,7 +205,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -248,10 +252,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_5.q.out b/ql/src/test/results/clientpositive/spark/union_remove_5.q.out index b31b3e6..7939e2c 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_5.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_5.q.out @@ -70,12 +70,13 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -156,10 +157,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -176,6 +177,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -190,7 +194,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -205,7 +209,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -256,10 +260,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 68 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_6.q.out b/ql/src/test/results/clientpositive/spark/union_remove_6.q.out index cd36189..2ab5042 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_6.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_6.q.out @@ -65,7 +65,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 Stage-1 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -176,6 +178,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-3 + Stats-Aggr Operator + Stage: Stage-1 Move Operator tables: @@ -186,6 +191,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 + Stage: Stage-4 + Stats-Aggr Operator + PREHOOK: query: FROM ( SELECT key, count(1) as `values` from inputTbl1 group by key UNION ALL diff --git a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out index 988973f..343afb9 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_6_subq.q.out @@ -73,7 +73,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 Stage-1 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -184,6 +186,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-3 + Stats-Aggr Operator + Stage: Stage-1 Move Operator tables: @@ -194,6 +199,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 + Stage: Stage-4 + Stats-Aggr Operator + PREHOOK: query: FROM ( select * from( SELECT key, count(1) as `values` from inputTbl1 group by key diff --git a/ql/src/test/results/clientpositive/spark/union_remove_7.q.out b/ql/src/test/results/clientpositive/spark/union_remove_7.q.out index 2a5eccf..4f4f509 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_7.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_7.q.out @@ -67,6 +67,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -161,6 +162,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -202,10 +206,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 336 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_8.q.out b/ql/src/test/results/clientpositive/spark/union_remove_8.q.out index 7aee2af..d2bbe57 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_8.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_8.q.out @@ -73,6 +73,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -163,6 +164,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -208,10 +212,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 350 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/union_remove_9.q.out b/ql/src/test/results/clientpositive/spark/union_remove_9.q.out index 87ceca2..2493da6 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_9.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_9.q.out @@ -74,12 +74,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -168,10 +169,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -188,6 +189,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Spark #### A masked pattern was here #### Vertices: @@ -198,7 +202,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Spark #### A masked pattern was here #### Vertices: @@ -209,7 +213,7 @@ STAGE PLANS: merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -264,10 +268,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 4 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 350 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats12.q.out b/ql/src/test/results/clientpositive/stats12.q.out index d5ca59b..f9a9f9d 100644 --- a/ql/src/test/results/clientpositive/stats12.q.out +++ b/ql/src/test/results/clientpositive/stats12.q.out @@ -78,7 +78,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -86,10 +86,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.analyze_srcpart numFiles 1 - numRows -1 + numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize -1 + rawDataSize 0 serialization.ddl struct analyze_srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -124,7 +124,7 @@ STAGE PLANS: ds 2008-04-08 hr 12 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -132,10 +132,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.analyze_srcpart numFiles 1 - numRows -1 + numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize -1 + rawDataSize 0 serialization.ddl struct analyze_srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -331,10 +331,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -373,10 +373,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats13.q.out b/ql/src/test/results/clientpositive/stats13.q.out index e34f0b7..b51eddd 100644 --- a/ql/src/test/results/clientpositive/stats13.q.out +++ b/ql/src/test/results/clientpositive/stats13.q.out @@ -79,7 +79,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -87,10 +87,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.analyze_srcpart numFiles 1 - numRows -1 + numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize -1 + rawDataSize 0 serialization.ddl struct analyze_srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -239,10 +239,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -281,10 +281,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -323,10 +323,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats2.q.out b/ql/src/test/results/clientpositive/stats2.q.out index 5e305d3..fd0047a 100644 --- a/ql/src/test/results/clientpositive/stats2.q.out +++ b/ql/src/test/results/clientpositive/stats2.q.out @@ -15,6 +15,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -49,6 +50,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.analyze_t1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/stats5.q.out b/ql/src/test/results/clientpositive/stats5.q.out index 484e834..824e786 100644 --- a/ql/src/test/results/clientpositive/stats5.q.out +++ b/ql/src/test/results/clientpositive/stats5.q.out @@ -22,7 +22,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_src - Statistics: Num rows: -1 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 Stats-Aggr Operator diff --git a/ql/src/test/results/clientpositive/stats6.q.out b/ql/src/test/results/clientpositive/stats6.q.out index b4435f2..67e9732 100644 --- a/ql/src/test/results/clientpositive/stats6.q.out +++ b/ql/src/test/results/clientpositive/stats6.q.out @@ -166,10 +166,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -208,10 +208,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats9.q.out b/ql/src/test/results/clientpositive/stats9.q.out index 8563f3b..b5bcc30 100644 --- a/ql/src/test/results/clientpositive/stats9.q.out +++ b/ql/src/test/results/clientpositive/stats9.q.out @@ -30,7 +30,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcbucket - Statistics: Num rows: -1 Data size: 11603 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 11603 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 Stats-Aggr Operator diff --git a/ql/src/test/results/clientpositive/stats_noscan_1.q.out b/ql/src/test/results/clientpositive/stats_noscan_1.q.out index 80c3092..9961d25 100644 --- a/ql/src/test/results/clientpositive/stats_noscan_1.q.out +++ b/ql/src/test/results/clientpositive/stats_noscan_1.q.out @@ -105,8 +105,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -147,8 +147,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -187,10 +187,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -229,10 +229,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -386,8 +386,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -428,8 +428,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -468,10 +468,10 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -510,10 +510,10 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out b/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out index 5488935..382bf13 100644 --- a/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out +++ b/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out @@ -78,10 +78,10 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5293 #### A masked pattern was here #### @@ -195,10 +195,10 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5293 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/insert_into1.q.out b/ql/src/test/results/clientpositive/tez/insert_into1.q.out index 359470b..7a01a83 100644 --- a/ql/src/test/results/clientpositive/tez/insert_into1.q.out +++ b/ql/src/test/results/clientpositive/tez/insert_into1.q.out @@ -367,6 +367,130 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@insert_into1 #### A masked pattern was here #### 10 +PREHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: insert_into1 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from insert_into1 +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from insert_into1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +11 PREHOOK: query: DROP TABLE insert_into1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@insert_into1 diff --git a/ql/src/test/results/clientpositive/tez/stats_noscan_1.q.out b/ql/src/test/results/clientpositive/tez/stats_noscan_1.q.out index aa49526..d8c0156 100644 --- a/ql/src/test/results/clientpositive/tez/stats_noscan_1.q.out +++ b/ql/src/test/results/clientpositive/tez/stats_noscan_1.q.out @@ -105,8 +105,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -147,8 +147,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -187,10 +187,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -229,10 +229,10 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -386,8 +386,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -428,8 +428,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -468,10 +468,10 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -510,10 +510,10 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_1.q.out b/ql/src/test/results/clientpositive/union_remove_1.q.out index 5fa8caf..f0fce2c 100644 --- a/ql/src/test/results/clientpositive/union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/union_remove_1.q.out @@ -62,8 +62,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -115,6 +116,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -193,10 +197,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_10.q.out b/ql/src/test/results/clientpositive/union_remove_10.q.out index 228ff9e..53505ed 100644 --- a/ql/src/test/results/clientpositive/union_remove_10.q.out +++ b/ql/src/test/results/clientpositive/union_remove_10.q.out @@ -78,14 +78,15 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 Stage-8 is a root stage + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -107,10 +108,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -127,26 +128,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -184,7 +188,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-8 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -252,10 +256,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 271 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_11.q.out b/ql/src/test/results/clientpositive/union_remove_11.q.out index bf38f0a..b1089a2 100644 --- a/ql/src/test/results/clientpositive/union_remove_11.q.out +++ b/ql/src/test/results/clientpositive/union_remove_11.q.out @@ -78,12 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -153,10 +154,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -173,20 +174,23 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true @@ -241,10 +245,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 115 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_12.q.out b/ql/src/test/results/clientpositive/union_remove_12.q.out index 3ac1d44..1143dd5 100644 --- a/ql/src/test/results/clientpositive/union_remove_12.q.out +++ b/ql/src/test/results/clientpositive/union_remove_12.q.out @@ -70,14 +70,15 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-10 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-11 is a root stage + Stage-10 depends on stages: Stage-11 STAGE PLANS: Stage: Stage-1 @@ -99,10 +100,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -119,26 +120,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -157,7 +161,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan @@ -232,10 +236,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 194 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_13.q.out b/ql/src/test/results/clientpositive/union_remove_13.q.out index b12d7cd..8d294a4 100644 --- a/ql/src/test/results/clientpositive/union_remove_13.q.out +++ b/ql/src/test/results/clientpositive/union_remove_13.q.out @@ -70,14 +70,15 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-10 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-11 is a root stage + Stage-10 depends on stages: Stage-11 STAGE PLANS: Stage: Stage-1 @@ -122,10 +123,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -142,26 +143,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -180,7 +184,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan @@ -255,10 +259,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 192 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_14.q.out b/ql/src/test/results/clientpositive/union_remove_14.q.out index 94f2ff5..657d768 100644 --- a/ql/src/test/results/clientpositive/union_remove_14.q.out +++ b/ql/src/test/results/clientpositive/union_remove_14.q.out @@ -72,14 +72,15 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-10 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-11 is a root stage + Stage-10 depends on stages: Stage-11 STAGE PLANS: Stage: Stage-1 @@ -101,10 +102,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -121,26 +122,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -159,7 +163,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan @@ -234,10 +238,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 194 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_15.q.out b/ql/src/test/results/clientpositive/union_remove_15.q.out index 67bdb1e..6195e90 100644 --- a/ql/src/test/results/clientpositive/union_remove_15.q.out +++ b/ql/src/test/results/clientpositive/union_remove_15.q.out @@ -68,8 +68,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -127,6 +128,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_16.q.out b/ql/src/test/results/clientpositive/union_remove_16.q.out index 2444337..7fddc22 100644 --- a/ql/src/test/results/clientpositive/union_remove_16.q.out +++ b/ql/src/test/results/clientpositive/union_remove_16.q.out @@ -66,13 +66,14 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-8 is a root stage STAGE PLANS: Stage: Stage-1 @@ -117,10 +118,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -139,26 +140,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_17.q.out b/ql/src/test/results/clientpositive/union_remove_17.q.out index a1b9474..3b058cd 100644 --- a/ql/src/test/results/clientpositive/union_remove_17.q.out +++ b/ql/src/test/results/clientpositive/union_remove_17.q.out @@ -63,6 +63,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -123,6 +124,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/union_remove_18.q.out b/ql/src/test/results/clientpositive/union_remove_18.q.out index 26e52e0..2b05810 100644 --- a/ql/src/test/results/clientpositive/union_remove_18.q.out +++ b/ql/src/test/results/clientpositive/union_remove_18.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -125,6 +126,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_19.q.out b/ql/src/test/results/clientpositive/union_remove_19.q.out index d8314f8..11504cd 100644 --- a/ql/src/test/results/clientpositive/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/union_remove_19.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -119,6 +120,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -197,10 +201,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### @@ -254,8 +258,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -310,6 +315,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -410,8 +418,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -470,6 +479,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_2.q.out b/ql/src/test/results/clientpositive/union_remove_2.q.out index d3522f2..8f48b2f 100644 --- a/ql/src/test/results/clientpositive/union_remove_2.q.out +++ b/ql/src/test/results/clientpositive/union_remove_2.q.out @@ -68,9 +68,10 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-2 depends on stages: Stage-0 Stage-3 is a root stage + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -122,6 +123,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -140,7 +144,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -204,10 +208,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 68 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_20.q.out b/ql/src/test/results/clientpositive/union_remove_20.q.out index 4503726..399850f 100644 --- a/ql/src/test/results/clientpositive/union_remove_20.q.out +++ b/ql/src/test/results/clientpositive/union_remove_20.q.out @@ -64,8 +64,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -121,6 +122,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -203,10 +207,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_21.q.out b/ql/src/test/results/clientpositive/union_remove_21.q.out index 98e0166..b4e49e4 100644 --- a/ql/src/test/results/clientpositive/union_remove_21.q.out +++ b/ql/src/test/results/clientpositive/union_remove_21.q.out @@ -64,8 +64,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -114,6 +115,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -187,10 +191,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 20 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_22.q.out b/ql/src/test/results/clientpositive/union_remove_22.q.out index 4f086f3..7a9545d 100644 --- a/ql/src/test/results/clientpositive/union_remove_22.q.out +++ b/ql/src/test/results/clientpositive/union_remove_22.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -123,6 +124,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -207,10 +211,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 60 #### A masked pattern was here #### @@ -262,8 +266,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -319,6 +324,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_23.q.out b/ql/src/test/results/clientpositive/union_remove_23.q.out index d0e21ba..02609d5 100644 --- a/ql/src/test/results/clientpositive/union_remove_23.q.out +++ b/ql/src/test/results/clientpositive/union_remove_23.q.out @@ -67,8 +67,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2, Stage-4 - Stage-4 is a root stage + Stage-0 depends on stages: Stage-2, Stage-5 + Stage-3 depends on stages: Stage-0 + Stage-5 is a root stage STAGE PLANS: Stage: Stage-1 @@ -154,7 +155,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -235,10 +239,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_24.q.out b/ql/src/test/results/clientpositive/union_remove_24.q.out index f2f6bba..10ef0a6 100644 --- a/ql/src/test/results/clientpositive/union_remove_24.q.out +++ b/ql/src/test/results/clientpositive/union_remove_24.q.out @@ -60,8 +60,9 @@ SELECT * FROM POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -117,6 +118,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -199,10 +203,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 60 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_25.q.out b/ql/src/test/results/clientpositive/union_remove_25.q.out index 3858bdc..7ddc10b 100644 --- a/ql/src/test/results/clientpositive/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/union_remove_25.q.out @@ -78,8 +78,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -133,6 +134,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -216,10 +220,10 @@ Table: outputtbl1 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### @@ -273,8 +277,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -328,6 +333,9 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -422,10 +430,10 @@ Table: outputtbl2 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 6812 #### A masked pattern was here #### @@ -457,8 +465,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -513,6 +522,9 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -612,10 +624,10 @@ Table: outputtbl3 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 6812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_3.q.out b/ql/src/test/results/clientpositive/union_remove_3.q.out index 40e554a..d0bddea 100644 --- a/ql/src/test/results/clientpositive/union_remove_3.q.out +++ b/ql/src/test/results/clientpositive/union_remove_3.q.out @@ -69,6 +69,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -148,6 +149,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( @@ -193,10 +197,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 72 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_4.q.out b/ql/src/test/results/clientpositive/union_remove_4.q.out index 10a8e89..39f40bf 100644 --- a/ql/src/test/results/clientpositive/union_remove_4.q.out +++ b/ql/src/test/results/clientpositive/union_remove_4.q.out @@ -62,13 +62,14 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-8 is a root stage STAGE PLANS: Stage: Stage-1 @@ -109,10 +110,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -129,6 +130,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -140,7 +144,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -152,13 +156,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -237,10 +241,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 40 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_5.q.out b/ql/src/test/results/clientpositive/union_remove_5.q.out index b59d6da..9804208 100644 --- a/ql/src/test/results/clientpositive/union_remove_5.q.out +++ b/ql/src/test/results/clientpositive/union_remove_5.q.out @@ -70,14 +70,15 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 Stage-8 is a root stage + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -118,10 +119,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -138,6 +139,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -149,7 +153,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -161,13 +165,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -186,7 +190,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-8 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan @@ -250,10 +254,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 68 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_6.q.out b/ql/src/test/results/clientpositive/union_remove_6.q.out index 67ba74c..b42ef69 100644 --- a/ql/src/test/results/clientpositive/union_remove_6.q.out +++ b/ql/src/test/results/clientpositive/union_remove_6.q.out @@ -64,10 +64,12 @@ insert overwrite table outputTbl2 select * POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2, Stage-6 Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 - Stage-4 is a root stage + Stage-5 depends on stages: Stage-1 + Stage-6 is a root stage STAGE PLANS: Stage: Stage-2 @@ -158,6 +160,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-4 + Stats-Aggr Operator + Stage: Stage-1 Move Operator tables: @@ -168,7 +173,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Stage: Stage-4 + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out index 0d1a436..c553ba8 100644 --- a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out +++ b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out @@ -72,10 +72,12 @@ insert overwrite table outputTbl2 select * POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2, Stage-6 Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 - Stage-4 is a root stage + Stage-5 depends on stages: Stage-1 + Stage-6 is a root stage STAGE PLANS: Stage: Stage-2 @@ -166,6 +168,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-4 + Stats-Aggr Operator + Stage: Stage-1 Move Operator tables: @@ -176,7 +181,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Stage: Stage-4 + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_7.q.out b/ql/src/test/results/clientpositive/union_remove_7.q.out index 76c678f..fcc3a59 100644 --- a/ql/src/test/results/clientpositive/union_remove_7.q.out +++ b/ql/src/test/results/clientpositive/union_remove_7.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -119,6 +120,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -197,10 +201,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 178 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_8.q.out b/ql/src/test/results/clientpositive/union_remove_8.q.out index fe67448..2927c00 100644 --- a/ql/src/test/results/clientpositive/union_remove_8.q.out +++ b/ql/src/test/results/clientpositive/union_remove_8.q.out @@ -72,9 +72,10 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-2 depends on stages: Stage-0 Stage-3 is a root stage + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -126,6 +127,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -144,7 +148,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -208,10 +212,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 3 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 271 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/union_remove_9.q.out b/ql/src/test/results/clientpositive/union_remove_9.q.out index 684091b..809ef21 100644 --- a/ql/src/test/results/clientpositive/union_remove_9.q.out +++ b/ql/src/test/results/clientpositive/union_remove_9.q.out @@ -74,13 +74,14 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-8 is a root stage STAGE PLANS: Stage: Stage-1 @@ -121,10 +122,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -141,26 +142,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -255,10 +259,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 - numRows -1 - rawDataSize -1 + numRows 0 + rawDataSize 0 totalSize 192 #### A masked pattern was here #### -- 1.7.12.4 (Apple Git-37)