From c4cef406d39403cda9ad077f367df487ce719c25 Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Fri, 22 May 2015 18:26:52 -0700 Subject: [PATCH] HIVE-10807 : Invalidate basic stats for insert queries if autogather=false --- .../org/apache/hadoop/hive/ql/QueryProperties.java | 10 -- .../org/apache/hadoop/hive/ql/exec/StatsTask.java | 8 +- .../hadoop/hive/ql/optimizer/GenMRFileSink1.java | 1 + .../hadoop/hive/ql/optimizer/GenMapRedUtils.java | 66 ++++----- .../apache/hadoop/hive/ql/parse/QBParseInfo.java | 9 -- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 10 +- ql/src/test/queries/clientpositive/insert_into1.q | 12 +- .../test/results/clientpositive/insert_into1.q.out | 155 +++++++++++++++++++++ .../clientpositive/skewjoin_union_remove_1.q.out | 20 ++- ql/src/test/results/clientpositive/stats1.q.out | 6 +- ql/src/test/results/clientpositive/stats11.q.out | 32 +++-- ql/src/test/results/clientpositive/stats12.q.out | 8 +- ql/src/test/results/clientpositive/stats13.q.out | 8 +- ql/src/test/results/clientpositive/stats18.q.out | 4 +- ql/src/test/results/clientpositive/stats2.q.out | 4 + ql/src/test/results/clientpositive/stats3.q.out | 6 +- ql/src/test/results/clientpositive/stats6.q.out | 4 +- .../results/clientpositive/stats_noscan_1.q.out | 8 +- .../results/clientpositive/stats_noscan_2.q.out | 2 +- .../clientpositive/stats_partscan_1_23.q.out | 4 +- .../results/clientpositive/union_remove_1.q.out | 8 +- .../results/clientpositive/union_remove_10.q.out | 28 ++-- .../results/clientpositive/union_remove_11.q.out | 22 +-- .../results/clientpositive/union_remove_12.q.out | 30 ++-- .../results/clientpositive/union_remove_13.q.out | 30 ++-- .../results/clientpositive/union_remove_14.q.out | 30 ++-- .../results/clientpositive/union_remove_15.q.out | 8 +- .../results/clientpositive/union_remove_16.q.out | 26 ++-- .../results/clientpositive/union_remove_17.q.out | 4 + .../results/clientpositive/union_remove_18.q.out | 8 +- .../results/clientpositive/union_remove_19.q.out | 24 +++- .../results/clientpositive/union_remove_2.q.out | 10 +- .../results/clientpositive/union_remove_20.q.out | 8 +- .../results/clientpositive/union_remove_21.q.out | 8 +- .../results/clientpositive/union_remove_22.q.out | 16 ++- .../results/clientpositive/union_remove_23.q.out | 10 +- .../results/clientpositive/union_remove_24.q.out | 8 +- .../results/clientpositive/union_remove_25.q.out | 30 ++-- .../results/clientpositive/union_remove_3.q.out | 4 + .../results/clientpositive/union_remove_4.q.out | 26 ++-- .../results/clientpositive/union_remove_5.q.out | 28 ++-- .../results/clientpositive/union_remove_6.q.out | 14 +- .../clientpositive/union_remove_6_subq.q.out | 14 +- .../results/clientpositive/union_remove_7.q.out | 8 +- .../results/clientpositive/union_remove_8.q.out | 10 +- .../results/clientpositive/union_remove_9.q.out | 26 ++-- 46 files changed, 553 insertions(+), 262 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index e8f7fba..3bc9432 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -39,7 +39,6 @@ boolean noScanAnalyzeCommand; boolean analyzeRewrite; boolean ctas; - boolean insertToTable; int outerQueryLimit; boolean hasJoin = false; @@ -115,14 +114,6 @@ public void setCTAS(boolean ctas) { this.ctas = ctas; } - public boolean isInsertToTable() { - return insertToTable; - } - - public void setInsertToTable(boolean insertToTable) { - this.insertToTable = insertToTable; - } - public int getOuterQueryLimit() { return outerQueryLimit; } @@ -276,7 +267,6 @@ public void clear() { noScanAnalyzeCommand = false; analyzeRewrite = false; ctas = false; - insertToTable = false; outerQueryLimit = -1; hasJoin = false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index 2a8167a..37611a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -174,13 +174,10 @@ private int aggregateStats() { if (statsAggregator != null) { String prefix = getAggregationPrefix(taskIndependent, table, null); updateStats(statsAggregator, parameters, prefix, maxPrefixLength, atomic); + parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE); } updateQuickStats(wh, parameters, tTable.getSd()); - - // write table stats to metastore - parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE); - db.alterTable(tableFullName, new Table(tTable)); console.printInfo("Table " + tableFullName + " stats: [" + toString(parameters) + ']'); @@ -333,8 +330,9 @@ private void updateQuickStats(Warehouse wh, Map parameters, private void clearStats(Map parameters) { for (String statType : StatsSetupConst.supportedStats) { if (parameters.containsKey(statType)) { - parameters.put(statType, "0"); + parameters.put(statType, "-1"); } + parameters.put(StatsSetupConst.COLUMN_STATS_ACCURATE, "false"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java index e5b9c2b..588909b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java @@ -61,6 +61,7 @@ public GenMRFileSink1() { * @param opProcCtx * context */ + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException { GenMRProcContext ctx = (GenMRProcContext) opProcCtx; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index acd9bf5..477a90f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -983,7 +983,7 @@ public static TableScanOperator createTemporaryFile( fileSinkOp.setParentOperators(Utilities.makeList(parent)); // Create a dummy TableScanOperator for the file generated through fileSinkOp - TableScanOperator tableScanOp = (TableScanOperator) createTemporaryTableScanOperator( + TableScanOperator tableScanOp = createTemporaryTableScanOperator( parent.getSchema()); // Connect this TableScanOperator to child. @@ -1419,37 +1419,42 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, assert statsWork != null : "Error when genereting StatsTask"; statsWork.setSourceTask(currTask); - statsWork.setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); - - if (currTask.getWork() instanceof MapredWork) { - MapredWork mrWork = (MapredWork) currTask.getWork(); - mrWork.getMapWork().setGatheringStats(true); - if (mrWork.getReduceWork() != null) { - mrWork.getReduceWork().setGatheringStats(true); - } - } else if (currTask.getWork() instanceof SparkWork) { - SparkWork work = (SparkWork) currTask.getWork(); - for (BaseWork w: work.getAllWork()) { - w.setGatheringStats(true); - } - } else { // must be TezWork - TezWork work = (TezWork) currTask.getWork(); - for (BaseWork w: work.getAllWork()) { - w.setGatheringStats(true); + if (hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)) { + statsWork.setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); + + if (currTask.getWork() instanceof MapredWork) { + MapredWork mrWork = (MapredWork) currTask.getWork(); + mrWork.getMapWork().setGatheringStats(true); + if (mrWork.getReduceWork() != null) { + mrWork.getReduceWork().setGatheringStats(true); + } + } else if (currTask.getWork() instanceof SparkWork) { + SparkWork work = (SparkWork) currTask.getWork(); + for (BaseWork w: work.getAllWork()) { + w.setGatheringStats(true); + } + } else { // must be TezWork + TezWork work = (TezWork) currTask.getWork(); + for (BaseWork w: work.getAllWork()) { + w.setGatheringStats(true); + } } - } - // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix - // in FileSinkDesc is used for stats publishing. They should be consistent. - statsWork.setAggKey(nd.getConf().getStatsAggPrefix()); - Task statsTask = TaskFactory.get(statsWork, hconf); + // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix + // in FileSinkDesc is used for stats publishing. They should be consistent. + statsWork.setAggKey(nd.getConf().getStatsAggPrefix()); - // mark the MapredWork and FileSinkOperator for gathering stats - nd.getConf().setGatherStats(true); - nd.getConf().setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); - nd.getConf().setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(hconf)); - // mrWork.addDestinationTable(nd.getConf().getTableInfo().getTableName()); + // mark the MapredWork and FileSinkOperator for gathering stats + nd.getConf().setGatherStats(true); + nd.getConf().setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); + nd.getConf().setMaxStatsKeyPrefixLength(StatsFactory.getMaxPrefixLength(hconf)); + } else { + statsWork.setClearAggregatorStats(true); + statsWork.setNoStatsAggregator(true); + statsWork.setStatsReliable(false); + } + Task statsTask = TaskFactory.get(statsWork, hconf); // subscribe feeds from the MoveTask so that MoveTask can forward the list // of dynamic partition list to the StatsTask mvTask.addDependentTask(statsTask); @@ -1464,8 +1469,7 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, * @return */ public static boolean isInsertInto(ParseContext parseCtx, FileSinkOperator fsOp) { - return fsOp.getConf().getTableInfo().getTableName() != null && - parseCtx.getQueryProperties().isInsertToTable(); + return fsOp.getConf().getTableInfo().getTableName() != null; } /** @@ -1687,7 +1691,7 @@ public static boolean isMergeRequired(List> mvTasks, HiveConf hco // no need of merging if the move is to a local file system MoveTask mvTask = (MoveTask) GenMapRedUtils.findMoveTask(mvTasks, fsOp); - if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)) { + if (mvTask != null && isInsertTable) { GenMapRedUtils.addStatsTask(fsOp, mvTask, currTask, hconf); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 14a7e9c..9072d7f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -65,7 +65,6 @@ private final HashSet insertIntoTables; private boolean isAnalyzeCommand; // used for the analyze command (statistics) - private boolean isInsertToTable; // used for insert overwrite command (statistics) private boolean isNoScanAnalyzeCommand; // used for the analyze command (statistics) (noscan) private boolean isPartialScanAnalyzeCommand; // used for the analyze command (statistics) // (partialscan) @@ -550,14 +549,6 @@ public boolean isAnalyzeCommand() { return isAnalyzeCommand; } - public void setIsInsertToTable(boolean isInsertToTable) { - this.isInsertToTable = isInsertToTable; - } - - public boolean isInsertToTable() { - return isInsertToTable; - } - public void addTableSpec(String tName, TableSpec tSpec) { tableSpecs.put(tName, tSpec); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7f355e5..dbac0b2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -730,7 +730,7 @@ private ASTNode genValuesTempTable(ASTNode originalFrom, QB qb) throws SemanticE Path dataDir = null; if(!qb.getEncryptedTargetTablePaths().isEmpty()) { //currently only Insert into T values(...) is supported thus only 1 values clause - //and only 1 target table are possible. If/when support for + //and only 1 target table are possible. If/when support for //select ... from values(...) is added an insert statement may have multiple //encrypted target tables. dataDir = ctx.getMRTmpPath(qb.getEncryptedTargetTablePaths().get(0).toUri()); @@ -1761,8 +1761,6 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException qb.getMetaData().setDestForAlias(name, ts.partHandle); } if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - // Set that variable to automatically collect stats during the MapReduce job - qb.getParseInfo().setIsInsertToTable(true); // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } @@ -1799,8 +1797,6 @@ public void getMetaData(QB qb, ReadEntity parentInput) throws SemanticException } if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { TableSpec ts = new TableSpec(db, conf, this.ast); - // Set that variable to automatically collect stats during the MapReduce job - qb.getParseInfo().setIsInsertToTable(true); // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts); } @@ -6379,7 +6375,6 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // verify that our destination is empty before proceeding if (dest_tab.isImmutable() && qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),dest_tab.getTableName())){ - qb.getParseInfo().isInsertToTable(); try { FileSystem fs = partPath.getFileSystem(conf); if (! MetaStoreUtils.isDirEmpty(fs,partPath)){ @@ -10070,7 +10065,7 @@ private void preProcessForInsert(ASTNode node, QB qb) throws SemanticException { return; } for (Node child : node.getChildren()) { - //each insert of multi insert looks like + //each insert of multi insert looks like //(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME T1))) if (((ASTNode) child).getToken().getType() != HiveParser.TOK_INSERT) { continue; @@ -12199,7 +12194,6 @@ private void copyInfoToQueryProperties(QueryProperties queryProperties) { queryProperties.setNoScanAnalyzeCommand(qb.getParseInfo().isNoScanAnalyzeCommand()); queryProperties.setAnalyzeRewrite(qb.isAnalyzeRewrite()); queryProperties.setCTAS(qb.getTableDesc() != null); - queryProperties.setInsertToTable(qb.getParseInfo().isInsertToTable()); queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() && !qb.getParseInfo().getDestToOrderBy().isEmpty()); queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit()); diff --git a/ql/src/test/queries/clientpositive/insert_into1.q b/ql/src/test/queries/clientpositive/insert_into1.q index f19506a..311963d 100644 --- a/ql/src/test/queries/clientpositive/insert_into1.q +++ b/ql/src/test/queries/clientpositive/insert_into1.q @@ -31,6 +31,16 @@ explain SELECT COUNT(*) FROM insert_into1; select count(*) from insert_into1; -DROP TABLE insert_into1; +set hive.stats.autogather=false; +explain +insert into table insert_into1 values(1, 'abc'); +insert into table insert_into1 values(1, 'abc'); +explain +SELECT COUNT(*) FROM insert_into1; +select count(*) from insert_into1; + + +DROP TABLE insert_into1; +set hive.stats.autogather=true; set hive.compute.query.using.stats=false; diff --git a/ql/src/test/results/clientpositive/insert_into1.q.out b/ql/src/test/results/clientpositive/insert_into1.q.out index 9e5f3bb..145bdac 100644 --- a/ql/src/test/results/clientpositive/insert_into1.q.out +++ b/ql/src/test/results/clientpositive/insert_into1.q.out @@ -337,6 +337,161 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@insert_into1 #### A masked pattern was here #### 10 +PREHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 6 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert into table insert_into1 values(1, 'abc') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@insert_into1 +POSTHOOK: query: insert into table insert_into1 values(1, 'abc') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@insert_into1 +POSTHOOK: Lineage: insert_into1.key EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: insert_into1.value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT COUNT(*) FROM insert_into1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: insert_into1 + Statistics: Num rows: -1 Data size: 120 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: -1 Data size: 120 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from insert_into1 +PREHOOK: type: QUERY +PREHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from insert_into1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@insert_into1 +#### A masked pattern was here #### +11 PREHOOK: query: DROP TABLE insert_into1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@insert_into1 diff --git a/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out b/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out index c67b681..211394b 100644 --- a/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out @@ -336,8 +336,9 @@ SELECT * FROM T1 a JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage + Stage-0 depends on stages: Stage-1, Stage-4 + Stage-2 depends on stages: Stage-0 + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -399,7 +400,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-3 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan @@ -487,8 +491,9 @@ SELECT * FROM T1 a RIGHT OUTER JOIN T2 b ON a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage + Stage-0 depends on stages: Stage-1, Stage-4 + Stage-2 depends on stages: Stage-0 + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -550,7 +555,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest1 - Stage: Stage-3 + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/stats1.q.out b/ql/src/test/results/clientpositive/stats1.q.out index 6f560d3..8dfce75 100644 --- a/ql/src/test/results/clientpositive/stats1.q.out +++ b/ql/src/test/results/clientpositive/stats1.q.out @@ -233,10 +233,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE true + COLUMN_STATS_ACCURATE false numFiles 3 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 1583 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats11.q.out b/ql/src/test/results/clientpositive/stats11.q.out index e51f049..bbc476e 100644 --- a/ql/src/test/results/clientpositive/stats11.q.out +++ b/ql/src/test/results/clientpositive/stats11.q.out @@ -91,8 +91,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 1 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 1358 #### A masked pattern was here #### @@ -140,8 +140,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 2 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 2750 #### A masked pattern was here #### @@ -189,8 +189,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 3 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 4200 #### A masked pattern was here #### @@ -238,8 +238,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 4 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 5812 #### A masked pattern was here #### @@ -405,10 +405,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 + numRows -1 partition_columns ds partition_columns.types string - rawDataSize 0 + rawDataSize -1 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -520,7 +520,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE true + COLUMN_STATS_ACCURATE false bucket_count 2 bucket_field_name key columns key,value @@ -529,6 +529,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 + numRows -1 + rawDataSize -1 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -539,7 +541,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: - COLUMN_STATS_ACCURATE true + COLUMN_STATS_ACCURATE false bucket_count 2 bucket_field_name key columns key,value @@ -548,6 +550,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 + numRows -1 + rawDataSize -1 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1027,10 +1031,10 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 + numRows -1 partition_columns ds partition_columns.types string - rawDataSize 0 + rawDataSize -1 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git a/ql/src/test/results/clientpositive/stats12.q.out b/ql/src/test/results/clientpositive/stats12.q.out index f8aa4d0..9c32761 100644 --- a/ql/src/test/results/clientpositive/stats12.q.out +++ b/ql/src/test/results/clientpositive/stats12.q.out @@ -78,7 +78,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -124,7 +124,7 @@ STAGE PLANS: ds 2008-04-08 hr 12 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -331,7 +331,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -373,7 +373,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 diff --git a/ql/src/test/results/clientpositive/stats13.q.out b/ql/src/test/results/clientpositive/stats13.q.out index 10ecfd2..85e6db3 100644 --- a/ql/src/test/results/clientpositive/stats13.q.out +++ b/ql/src/test/results/clientpositive/stats13.q.out @@ -79,7 +79,7 @@ STAGE PLANS: ds 2008-04-08 hr 11 properties: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true bucket_count -1 columns key,value columns.comments 'default','default' @@ -239,7 +239,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -281,7 +281,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -323,7 +323,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 diff --git a/ql/src/test/results/clientpositive/stats18.q.out b/ql/src/test/results/clientpositive/stats18.q.out index a061846..64a765d 100644 --- a/ql/src/test/results/clientpositive/stats18.q.out +++ b/ql/src/test/results/clientpositive/stats18.q.out @@ -99,8 +99,8 @@ Protect Mode: None Partition Parameters: COLUMN_STATS_ACCURATE true numFiles 2 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 7170 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats2.q.out b/ql/src/test/results/clientpositive/stats2.q.out index 694c1a2..ac4ea05 100644 --- a/ql/src/test/results/clientpositive/stats2.q.out +++ b/ql/src/test/results/clientpositive/stats2.q.out @@ -15,6 +15,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -49,6 +50,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.analyze_t1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table analyze_t1 partition (ds, hr) select * from srcpart where ds is not null PREHOOK: type: QUERY PREHOOK: Input: default@srcpart diff --git a/ql/src/test/results/clientpositive/stats3.q.out b/ql/src/test/results/clientpositive/stats3.q.out index dd3a95b..23fdeca 100644 --- a/ql/src/test/results/clientpositive/stats3.q.out +++ b/ql/src/test/results/clientpositive/stats3.q.out @@ -87,10 +87,10 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE true + COLUMN_STATS_ACCURATE false numFiles 1 - numRows 0 - rawDataSize 0 + numRows -1 + rawDataSize -1 totalSize 11 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats6.q.out b/ql/src/test/results/clientpositive/stats6.q.out index b4435f2..e6f4986 100644 --- a/ql/src/test/results/clientpositive/stats6.q.out +++ b/ql/src/test/results/clientpositive/stats6.q.out @@ -166,7 +166,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -208,7 +208,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 diff --git a/ql/src/test/results/clientpositive/stats_noscan_1.q.out b/ql/src/test/results/clientpositive/stats_noscan_1.q.out index 80c3092..6637881 100644 --- a/ql/src/test/results/clientpositive/stats_noscan_1.q.out +++ b/ql/src/test/results/clientpositive/stats_noscan_1.q.out @@ -187,7 +187,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -229,7 +229,7 @@ Table: analyze_srcpart Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -468,7 +468,7 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -510,7 +510,7 @@ Table: analyze_srcpart_partial Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 diff --git a/ql/src/test/results/clientpositive/stats_noscan_2.q.out b/ql/src/test/results/clientpositive/stats_noscan_2.q.out index 238727d..9ac9bb5 100644 --- a/ql/src/test/results/clientpositive/stats_noscan_2.q.out +++ b/ql/src/test/results/clientpositive/stats_noscan_2.q.out @@ -52,7 +52,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE true + COLUMN_STATS_ACCURATE false EXTERNAL TRUE numFiles 0 numRows -1 diff --git a/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out b/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out index 5488935..eb85c6f 100644 --- a/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out +++ b/ql/src/test/results/clientpositive/stats_partscan_1_23.q.out @@ -78,7 +78,7 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 @@ -195,7 +195,7 @@ Table: analyze_srcpart_partial_scan Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 1 numRows -1 rawDataSize -1 diff --git a/ql/src/test/results/clientpositive/union_remove_1.q.out b/ql/src/test/results/clientpositive/union_remove_1.q.out index f30f39b..995b765 100644 --- a/ql/src/test/results/clientpositive/union_remove_1.q.out +++ b/ql/src/test/results/clientpositive/union_remove_1.q.out @@ -62,8 +62,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -115,6 +116,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_10.q.out b/ql/src/test/results/clientpositive/union_remove_10.q.out index e720c37..28f60a8 100644 --- a/ql/src/test/results/clientpositive/union_remove_10.q.out +++ b/ql/src/test/results/clientpositive/union_remove_10.q.out @@ -78,14 +78,15 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 Stage-8 is a root stage + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -107,10 +108,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -127,26 +128,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -184,7 +188,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-8 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_11.q.out b/ql/src/test/results/clientpositive/union_remove_11.q.out index 571b398..c437c39 100644 --- a/ql/src/test/results/clientpositive/union_remove_11.q.out +++ b/ql/src/test/results/clientpositive/union_remove_11.q.out @@ -78,12 +78,13 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -153,10 +154,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -173,20 +174,23 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true diff --git a/ql/src/test/results/clientpositive/union_remove_12.q.out b/ql/src/test/results/clientpositive/union_remove_12.q.out index 4d6f4a1..1fbe35a 100644 --- a/ql/src/test/results/clientpositive/union_remove_12.q.out +++ b/ql/src/test/results/clientpositive/union_remove_12.q.out @@ -70,14 +70,15 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-10 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-11 is a root stage + Stage-10 depends on stages: Stage-11 STAGE PLANS: Stage: Stage-1 @@ -99,10 +100,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -119,26 +120,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -157,7 +161,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_13.q.out b/ql/src/test/results/clientpositive/union_remove_13.q.out index a558967..0c0f6f9 100644 --- a/ql/src/test/results/clientpositive/union_remove_13.q.out +++ b/ql/src/test/results/clientpositive/union_remove_13.q.out @@ -70,14 +70,15 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-10 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-11 is a root stage + Stage-10 depends on stages: Stage-11 STAGE PLANS: Stage: Stage-1 @@ -122,10 +123,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -142,26 +143,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -180,7 +184,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_14.q.out b/ql/src/test/results/clientpositive/union_remove_14.q.out index 1502749..bc74d9b 100644 --- a/ql/src/test/results/clientpositive/union_remove_14.q.out +++ b/ql/src/test/results/clientpositive/union_remove_14.q.out @@ -72,14 +72,15 @@ FROM inputTbl1 a join inputTbl1 b on a.key=b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-9 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-10 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-10 is a root stage - Stage-9 depends on stages: Stage-10 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-11 is a root stage + Stage-10 depends on stages: Stage-11 STAGE PLANS: Stage: Stage-1 @@ -101,10 +102,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -121,26 +122,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-10 + Stage: Stage-11 Map Reduce Local Work Alias -> Map Local Tables: null-subquery2:c-subquery2:a @@ -159,7 +163,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) - Stage: Stage-9 + Stage: Stage-10 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_15.q.out b/ql/src/test/results/clientpositive/union_remove_15.q.out index 2b3c109..0da42ca 100644 --- a/ql/src/test/results/clientpositive/union_remove_15.q.out +++ b/ql/src/test/results/clientpositive/union_remove_15.q.out @@ -68,8 +68,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -127,6 +128,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_16.q.out b/ql/src/test/results/clientpositive/union_remove_16.q.out index ef32e1c..d504899 100644 --- a/ql/src/test/results/clientpositive/union_remove_16.q.out +++ b/ql/src/test/results/clientpositive/union_remove_16.q.out @@ -66,13 +66,14 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-8 is a root stage STAGE PLANS: Stage: Stage-1 @@ -117,10 +118,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -139,26 +140,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_17.q.out b/ql/src/test/results/clientpositive/union_remove_17.q.out index 56b22d9..ef374db 100644 --- a/ql/src/test/results/clientpositive/union_remove_17.q.out +++ b/ql/src/test/results/clientpositive/union_remove_17.q.out @@ -63,6 +63,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -123,6 +124,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 partition (ds) SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/union_remove_18.q.out b/ql/src/test/results/clientpositive/union_remove_18.q.out index 80971dd..7b85c4a 100644 --- a/ql/src/test/results/clientpositive/union_remove_18.q.out +++ b/ql/src/test/results/clientpositive/union_remove_18.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -125,6 +126,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_19.q.out b/ql/src/test/results/clientpositive/union_remove_19.q.out index 030c5ce..62ee4ed 100644 --- a/ql/src/test/results/clientpositive/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/union_remove_19.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -119,6 +120,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -254,8 +258,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -310,6 +315,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -410,8 +418,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -470,6 +479,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_2.q.out b/ql/src/test/results/clientpositive/union_remove_2.q.out index 1593029..e221c9b 100644 --- a/ql/src/test/results/clientpositive/union_remove_2.q.out +++ b/ql/src/test/results/clientpositive/union_remove_2.q.out @@ -68,9 +68,10 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-2 depends on stages: Stage-0 Stage-3 is a root stage + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -122,6 +123,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -140,7 +144,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_20.q.out b/ql/src/test/results/clientpositive/union_remove_20.q.out index fc8464d..17c8fd1 100644 --- a/ql/src/test/results/clientpositive/union_remove_20.q.out +++ b/ql/src/test/results/clientpositive/union_remove_20.q.out @@ -64,8 +64,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -121,6 +122,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_21.q.out b/ql/src/test/results/clientpositive/union_remove_21.q.out index 842e67f..4b39cd0 100644 --- a/ql/src/test/results/clientpositive/union_remove_21.q.out +++ b/ql/src/test/results/clientpositive/union_remove_21.q.out @@ -64,8 +64,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -114,6 +115,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_22.q.out b/ql/src/test/results/clientpositive/union_remove_22.q.out index 71054ae..2a45711 100644 --- a/ql/src/test/results/clientpositive/union_remove_22.q.out +++ b/ql/src/test/results/clientpositive/union_remove_22.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -123,6 +124,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -262,8 +266,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -319,6 +324,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_23.q.out b/ql/src/test/results/clientpositive/union_remove_23.q.out index 96e7d3d..fcea41f 100644 --- a/ql/src/test/results/clientpositive/union_remove_23.q.out +++ b/ql/src/test/results/clientpositive/union_remove_23.q.out @@ -67,8 +67,9 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2, Stage-4 - Stage-4 is a root stage + Stage-0 depends on stages: Stage-2, Stage-5 + Stage-3 depends on stages: Stage-0 + Stage-5 is a root stage STAGE PLANS: Stage: Stage-1 @@ -154,7 +155,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-3 + Stats-Aggr Operator + + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_24.q.out b/ql/src/test/results/clientpositive/union_remove_24.q.out index d9e86bf..ccbc1e5 100644 --- a/ql/src/test/results/clientpositive/union_remove_24.q.out +++ b/ql/src/test/results/clientpositive/union_remove_24.q.out @@ -60,8 +60,9 @@ SELECT * FROM POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -117,6 +118,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_25.q.out b/ql/src/test/results/clientpositive/union_remove_25.q.out index 0435ad5..ec06ca8 100644 --- a/ql/src/test/results/clientpositive/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/union_remove_25.q.out @@ -78,8 +78,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -133,6 +134,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -216,7 +220,7 @@ Table: outputtbl1 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 numRows -1 rawDataSize -1 @@ -273,8 +277,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -328,6 +333,9 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -422,7 +430,7 @@ Table: outputtbl2 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 numRows -1 rawDataSize -1 @@ -457,8 +465,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -513,6 +522,9 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -612,7 +624,7 @@ Table: outputtbl3 Protect Mode: None #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE false + COLUMN_STATS_ACCURATE true numFiles 2 numRows -1 rawDataSize -1 diff --git a/ql/src/test/results/clientpositive/union_remove_3.q.out b/ql/src/test/results/clientpositive/union_remove_3.q.out index 0de46d0..01a0685 100644 --- a/ql/src/test/results/clientpositive/union_remove_3.q.out +++ b/ql/src/test/results/clientpositive/union_remove_3.q.out @@ -69,6 +69,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -148,6 +149,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-2 + Stats-Aggr Operator + PREHOOK: query: insert overwrite table outputTbl1 SELECT * FROM ( diff --git a/ql/src/test/results/clientpositive/union_remove_4.q.out b/ql/src/test/results/clientpositive/union_remove_4.q.out index b83a0a8..c9e3d4f 100644 --- a/ql/src/test/results/clientpositive/union_remove_4.q.out +++ b/ql/src/test/results/clientpositive/union_remove_4.q.out @@ -62,13 +62,14 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-8 is a root stage STAGE PLANS: Stage: Stage-1 @@ -109,10 +110,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -129,6 +130,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -140,7 +144,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -152,13 +156,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_5.q.out b/ql/src/test/results/clientpositive/union_remove_5.q.out index 81e64b4..ef70efb 100644 --- a/ql/src/test/results/clientpositive/union_remove_5.q.out +++ b/ql/src/test/results/clientpositive/union_remove_5.q.out @@ -70,14 +70,15 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7, Stage-8 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8, Stage-9 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 Stage-8 is a root stage + Stage-9 is a root stage STAGE PLANS: Stage: Stage-1 @@ -118,10 +119,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -138,6 +139,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -149,7 +153,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-4 + Stage: Stage-5 Map Reduce Map Operator Tree: TableScan @@ -161,13 +165,13 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan @@ -186,7 +190,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 - Stage: Stage-8 + Stage: Stage-9 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_6.q.out b/ql/src/test/results/clientpositive/union_remove_6.q.out index 215e90e..cd67662 100644 --- a/ql/src/test/results/clientpositive/union_remove_6.q.out +++ b/ql/src/test/results/clientpositive/union_remove_6.q.out @@ -64,10 +64,12 @@ insert overwrite table outputTbl2 select * POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2, Stage-6 Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 - Stage-4 is a root stage + Stage-5 depends on stages: Stage-1 + Stage-6 is a root stage STAGE PLANS: Stage: Stage-2 @@ -158,6 +160,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-4 + Stats-Aggr Operator + Stage: Stage-1 Move Operator tables: @@ -168,7 +173,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Stage: Stage-4 + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out index 2661bcb..209a5bb 100644 --- a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out +++ b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out @@ -72,10 +72,12 @@ insert overwrite table outputTbl2 select * POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-3 depends on stages: Stage-2, Stage-4 + Stage-3 depends on stages: Stage-2, Stage-6 Stage-0 depends on stages: Stage-3 + Stage-4 depends on stages: Stage-0 Stage-1 depends on stages: Stage-3 - Stage-4 is a root stage + Stage-5 depends on stages: Stage-1 + Stage-6 is a root stage STAGE PLANS: Stage: Stage-2 @@ -166,6 +168,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl1 + Stage: Stage-4 + Stats-Aggr Operator + Stage: Stage-1 Move Operator tables: @@ -176,7 +181,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.outputtbl2 - Stage: Stage-4 + Stage: Stage-5 + Stats-Aggr Operator + + Stage: Stage-6 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_7.q.out b/ql/src/test/results/clientpositive/union_remove_7.q.out index 4feb092..e36e7ea 100644 --- a/ql/src/test/results/clientpositive/union_remove_7.q.out +++ b/ql/src/test/results/clientpositive/union_remove_7.q.out @@ -66,8 +66,9 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 is a root stage STAGE PLANS: Stage: Stage-1 @@ -119,6 +120,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_8.q.out b/ql/src/test/results/clientpositive/union_remove_8.q.out index 5d22aa5..8f174b1 100644 --- a/ql/src/test/results/clientpositive/union_remove_8.q.out +++ b/ql/src/test/results/clientpositive/union_remove_8.q.out @@ -72,9 +72,10 @@ FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1, Stage-2, Stage-3 - Stage-2 is a root stage + Stage-0 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-2 depends on stages: Stage-0 Stage-3 is a root stage + Stage-4 is a root stage STAGE PLANS: Stage: Stage-1 @@ -126,6 +127,9 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Map Reduce Map Operator Tree: TableScan @@ -144,7 +148,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan diff --git a/ql/src/test/results/clientpositive/union_remove_9.q.out b/ql/src/test/results/clientpositive/union_remove_9.q.out index 7380d12..152b3f1 100644 --- a/ql/src/test/results/clientpositive/union_remove_9.q.out +++ b/ql/src/test/results/clientpositive/union_remove_9.q.out @@ -74,13 +74,14 @@ select * FROM ( POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-6 depends on stages: Stage-1, Stage-7 , consists of Stage-3, Stage-2, Stage-4 - Stage-3 - Stage-0 depends on stages: Stage-3, Stage-2, Stage-5 - Stage-2 + Stage-7 depends on stages: Stage-1, Stage-8 , consists of Stage-4, Stage-3, Stage-5 Stage-4 - Stage-5 depends on stages: Stage-4 - Stage-7 is a root stage + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + Stage-8 is a root stage STAGE PLANS: Stage: Stage-1 @@ -121,10 +122,10 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.outputtbl1 - Stage: Stage-6 + Stage: Stage-7 Conditional Operator - Stage: Stage-3 + Stage: Stage-4 Move Operator files: hdfs directory: true @@ -141,26 +142,29 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-4 + Stage: Stage-5 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-7 + Stage: Stage-8 Map Reduce Map Operator Tree: TableScan -- 1.7.12.4 (Apple Git-37)