diff --git common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java index 7c27d07024..45f1afec5d 100644 --- common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@ -257,6 +257,14 @@ public static void setColumnStatsState(Map params, List } } + public static boolean canColumnStatsMerge(Map params, String colName) { + if (params == null) { + return false; + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + return stats.columnStats.containsKey(colName); + } + public static void clearColumnStatsState(Map params) { if (params == null) { return; @@ -294,7 +302,9 @@ public static void setStatsStateForCreateTable(Map params, } } setBasicStatsState(params, setting); - setColumnStatsState(params, cols); + if (TRUE.equals(setting)) { + setColumnStatsState(params, cols); + } } private static ColumnStatsAccurate parseStatsAcc(String statsAcc) { diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java index b7dc88c939..bcbfe977dd 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -249,7 +249,8 @@ public void print(Printer printer, int indentFlag, String type, Vertex callingVe // find the right op Op choose = null; for (Op op : this.outputOps) { - if (op.outputVertexName.equals(callingVertex.name)) { + // op.outputVertexName may be null + if (callingVertex.name.equals(op.outputVertexName)) { choose = op; } } diff --git contrib/src/test/results/clientpositive/serde_typedbytes.q.out contrib/src/test/results/clientpositive/serde_typedbytes.q.out index 6876ca8775..c844a70627 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git contrib/src/test/results/clientpositive/serde_typedbytes2.q.out contrib/src/test/results/clientpositive/serde_typedbytes2.q.out index 79cf8fe1e5..c0228aa4ff 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes2.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes2.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git contrib/src/test/results/clientpositive/serde_typedbytes3.q.out contrib/src/test/results/clientpositive/serde_typedbytes3.q.out index fec58ef026..0b7541ac13 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes3.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes3.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git contrib/src/test/results/clientpositive/serde_typedbytes4.q.out contrib/src/test/results/clientpositive/serde_typedbytes4.q.out index 1131478a7b..981ff21bb3 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes4.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes4.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git contrib/src/test/results/clientpositive/serde_typedbytes5.q.out contrib/src/test/results/clientpositive/serde_typedbytes5.q.out index 8d3b95ece8..5a7df3c837 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes5.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes5.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out index 68a417d0c1..acb576eb5c 100644 --- hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out +++ hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.src_x1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index e55b1c257e..2025ad882a 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -109,6 +109,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1,tmp_values_col2 @@ -116,14 +117,19 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1,tmp_values_col2 @@ -131,9 +137,13 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.values__tmp__table__3 name: default.values__tmp__table__3 @@ -208,8 +218,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out index f50f4af817..c465d79a3b 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out @@ -104,6 +104,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1 @@ -111,14 +112,19 @@ STAGE PLANS: columns.types string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1 @@ -126,9 +132,13 @@ STAGE PLANS: columns.types string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.values__tmp__table__3 name: default.values__tmp__table__3 @@ -168,8 +178,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### Stage: Stage-3 Map Reduce diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index 660cebba5f..39b304b536 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -127,6 +127,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1,tmp_values_col2 @@ -134,14 +135,19 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1,tmp_values_col2 @@ -149,9 +155,13 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.values__tmp__table__3 name: default.values__tmp__table__3 @@ -226,8 +236,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out index ba0e83d562..27c4b795ef 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out @@ -112,6 +112,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1 @@ -119,14 +120,19 @@ STAGE PLANS: columns.types string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true"}} bucket_count -1 column.name.delimiter , columns tmp_values_col1 @@ -134,9 +140,13 @@ STAGE PLANS: columns.types string #### A masked pattern was here #### name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 serialization.ddl struct values__tmp__table__3 { string tmp_values_col1} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.values__tmp__table__3 name: default.values__tmp__table__3 @@ -176,8 +186,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### Stage: Stage-3 Map Reduce diff --git itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out index 2ababb1eec..45e6d25b03 100644 --- itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out @@ -245,7 +245,8 @@ STAGE PLANS: name: default.blobstore_table Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: EXPLAIN EXTENDED FROM hdfs_table INSERT OVERWRITE TABLE blobstore_table SELECT hdfs_table.key GROUP BY hdfs_table.key ORDER BY hdfs_table.key @@ -469,8 +470,9 @@ STAGE PLANS: name: default.blobstore_table Stage: Stage-3 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### PREHOOK: query: DROP TABLE hdfs_table PREHOOK: type: DROPTABLE diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java index ad2baa2e26..e8ef4b97d6 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java @@ -44,6 +44,7 @@ public void testMTQueries1() throws Exception { util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true); util.getConf().set("hive.stats.dbclass", "fs"); util.getConf().set("hive.mapred.mode", "nonstrict"); + util.getConf().set("hive.stats.column.autogather", "false"); } boolean success = QTestUtil.queryListRunnerMultiThreaded(qfiles, qts); if (!success) { diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 4a9af80fdc..96173c014e 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -201,6 +201,7 @@ private static void startMiniHS2(HiveConf conf) throws Exception { private static void startMiniHS2(HiveConf conf, boolean httpMode) throws Exception { conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); + conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false); MiniHS2.Builder builder = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false); if (httpMode) { builder = builder.withHTTPTransport(); diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index d472bb3f9e..7bbf2d7ad8 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -111,6 +111,9 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ auto_sortmerge_join_7.q,\ auto_sortmerge_join_8.q,\ auto_sortmerge_join_9.q,\ + autoColumnStats_1.q,\ + autoColumnStats_10.q,\ + autoColumnStats_2.q,\ bucket2.q,\ bucket3.q,\ bucket4.q,\ @@ -472,8 +475,6 @@ minillaplocal.query.files=\ auto_sortmerge_join_6.q,\ auto_sortmerge_join_8.q,\ auto_sortmerge_join_9.q,\ - autoColumnStats_1.q,\ - autoColumnStats_2.q,\ bucket4.q,\ bucket_groupby.q,\ bucket_many.q,\ diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index b279e1d567..3dbe72fef4 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -248,7 +248,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, part.setDbName(newDbName); part.setTableName(newTblName); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, - part.getValues(), part.getSd().getCols(), oldt, part); + part.getValues(), part.getSd().getCols(), oldt, part, null); if (colStats != null) { columnStatsNeedUpdated.put(part, colStats); } @@ -287,7 +287,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, List oldCols = part.getSd().getCols(); part.getSd().setCols(newt.getSd().getCols()); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, - part.getValues(), oldCols, oldt, part); + part.getValues(), oldCols, oldt, part, null); assert(colStats == null); msdb.alterPartition(dbname, name, part.getValues(), part); } @@ -296,6 +296,17 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, LOG.warn("Alter table does not cascade changes to its partitions."); } } else { + if (isPartitionedTable + && !MetaStoreUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) { + parts = msdb.getPartitions(dbname, name, -1); + for (Partition part : parts) { + List oldCols = part.getSd().getCols(); + ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, + part.getValues(), oldCols, oldt, part, newt.getSd().getCols()); + assert (colStats == null); + msdb.alterPartition(dbname, name, part.getValues(), part); + } + } alterTableUpdateTableColumnStats(msdb, oldt, newt); } } @@ -412,7 +423,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String // PartitionView does not have SD. We do not need update its column stats if (oldPart.getSd() != null) { updateOrGetPartitionColumnStats(msdb, dbname, name, new_part.getValues(), - oldPart.getSd().getCols(), tbl, new_part); + oldPart.getSd().getCols(), tbl, new_part, null); } msdb.alterPartition(dbname, name, new_part.getValues(), new_part); if (transactionalListeners != null && !transactionalListeners.isEmpty()) { @@ -539,7 +550,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()); ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, dbname, name, oldPart.getValues(), - oldPart.getSd().getCols(), tbl, new_part); + oldPart.getSd().getCols(), tbl, new_part, null); msdb.alterPartition(dbname, name, part_vals, new_part); if (cs != null) { cs.getStatsDesc().setPartName(newPartName); @@ -637,7 +648,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String // PartitionView does not have SD and we do not need to update its column stats if (oldTmpPart.getSd() != null) { updateOrGetPartitionColumnStats(msdb, dbname, name, oldTmpPart.getValues(), - oldTmpPart.getSd().getCols(), tbl, tmpPart); + oldTmpPart.getSd().getCols(), tbl, tmpPart, null); } } @@ -789,12 +800,14 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa private ColumnStatistics updateOrGetPartitionColumnStats( RawStore msdb, String dbname, String tblname, List partVals, - List oldCols, Table table, Partition part) + List oldCols, Table table, Partition part, List newCols) throws MetaException, InvalidObjectException { ColumnStatistics newPartsColStats = null; try { - List newCols = part.getSd() == null ? - new ArrayList() : part.getSd().getCols(); + // if newCols are not specified, use default ones. + if (newCols == null) { + newCols = part.getSd() == null ? new ArrayList() : part.getSd().getCols(); + } String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals); String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname) diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 53cb3d987f..47633a1b32 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -35,6 +35,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -63,6 +64,7 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; + import org.apache.commons.cli.OptionBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -6971,39 +6973,80 @@ public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) if (request.isSetNeedMerge() && request.isNeedMerge()) { // one single call to get all column stats ColumnStatistics csOld = getMS().getTableColumnStatistics(dbName, tableName, colNames); - if (csOld != null && csOld.getStatsObjSize() != 0) { + Table t = getTable(dbName, tableName); + // we first use t.getParameters() to prune the stats + MetaStoreUtils.pruneColumnStats(firstColStats, t.getParameters()); + // we merge those that can be merged + if (csOld != null && csOld.getStatsObjSize() != 0 + && !firstColStats.getStatsObj().isEmpty()) { MetaStoreUtils.mergeColStats(firstColStats, csOld); } + if (!firstColStats.getStatsObj().isEmpty()) { + return update_table_column_statistics(firstColStats); + } else { + LOG.debug("All the column stats are not accurate to merge."); + return true; + } + } else { + // This is the overwrite case, we do not care about the accuracy. + return update_table_column_statistics(firstColStats); } - return update_table_column_statistics(firstColStats); } } else { // partition level column stats merging - List partitionNames = new ArrayList<>(); + List partitions = new ArrayList<>(); + // note that we may have two or more duplicate partition names. + // see autoColumnStats_2.q under TestMiniLlapLocalCliDriver + Map newStatsMap = new HashMap<>(); for (ColumnStatistics csNew : csNews) { - partitionNames.add(csNew.getStatsDesc().getPartName()); + String partName = csNew.getStatsDesc().getPartName(); + if (newStatsMap.containsKey(partName)) { + MetaStoreUtils.mergeColStats(csNew, newStatsMap.get(partName)); + } + newStatsMap.put(partName, csNew); } - Map map = new HashMap<>(); + + Map oldStatsMap = new HashMap<>(); + Map mapToPart = new HashMap<>(); if (request.isSetNeedMerge() && request.isNeedMerge()) { // a single call to get all column stats for all partitions + List partitionNames = new ArrayList<>(); + partitionNames.addAll(newStatsMap.keySet()); List csOlds = getMS().getPartitionColumnStatistics(dbName, tableName, partitionNames, colNames); - if (csNews.size() != csOlds.size()) { + if (newStatsMap.values().size() != csOlds.size()) { // some of the partitions miss stats. LOG.debug("Some of the partitions miss stats."); } for (ColumnStatistics csOld : csOlds) { - map.put(csOld.getStatsDesc().getPartName(), csOld); + oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld); + } + // another single call to get all the partition objects + partitions = getMS().getPartitionsByNames(dbName, tableName, partitionNames); + for (int index = 0; index < partitionNames.size(); index++) { + mapToPart.put(partitionNames.get(index), partitions.get(index)); } } Table t = getTable(dbName, tableName); - for (int index = 0; index < csNews.size(); index++) { - ColumnStatistics csNew = csNews.get(index); - ColumnStatistics csOld = map.get(csNew.getStatsDesc().getPartName()); - if (csOld != null && csOld.getStatsObjSize() != 0) { - MetaStoreUtils.mergeColStats(csNew, csOld); + for (Entry entry : newStatsMap.entrySet()) { + ColumnStatistics csNew = entry.getValue(); + ColumnStatistics csOld = oldStatsMap.get(entry.getKey()); + if (request.isSetNeedMerge() && request.isNeedMerge()) { + // we first use getParameters() to prune the stats + MetaStoreUtils.pruneColumnStats(csNew, mapToPart.get(entry.getKey()).getParameters()); + // we merge those that can be merged + if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) { + MetaStoreUtils.mergeColStats(csNew, csOld); + } + if (!csNew.getStatsObj().isEmpty()) { + ret = ret && updatePartitonColStats(t, csNew); + } else { + LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName() + + " are not accurate to merge."); + } + } else { + ret = ret && updatePartitonColStats(t, csNew); } - ret = ret && updatePartitonColStats(t, csNew); } } return ret; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index bbe13fd77b..f2cd1d7895 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1903,6 +1903,19 @@ public static String encodeTableName(String name) { return sb.toString(); } + public static void pruneColumnStats(ColumnStatistics csNew, Map parameters) { + List list = new ArrayList<>(); + for (int index = 0; index < csNew.getStatsObj().size(); index++) { + ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index); + // canColumnStatsMerge guarantees that it is accurate before we do merge + if (StatsSetupConst.canColumnStatsMerge(parameters, statsObjNew.getColName())) { + list.add(statsObjNew); + } + // in all the other cases, we can not merge + } + csNew.setStatsObj(list); + } + // this function will merge csOld into csNew. public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) throws InvalidObjectException { @@ -1926,13 +1939,20 @@ public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index); ColumnStatisticsObj statsObjOld = map.get(statsObjNew.getColName()); if (statsObjOld != null) { + // because we already confirm that the stats is accurate + // it is impossible that the column types have been changed while the + // column stats is still accurate. + assert (statsObjNew.getStatsData().getSetField() == statsObjOld.getStatsData() + .getSetField()); // If statsObjOld is found, we can merge. ColumnStatsMerger merger = ColumnStatsMergerFactory.getColumnStatsMerger(statsObjNew, statsObjOld); merger.merge(statsObjNew, statsObjOld); } + // If statsObjOld is not found, we just use statsObjNew as it is accurate. list.add(statsObjNew); } + // in all the other cases, we can not merge csNew.setStatsObj(list); } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 3053dcb50b..baeee8be71 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -7081,7 +7081,7 @@ public boolean updateTableColumnStatistics(ColumnStatistics colStats) MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics( ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj); writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); - colNames.add(statsObj.getColName()); + // There is no need to add colname again, otherwise we will get duplicate colNames. } // Set the table properties diff --git metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java index 66be524139..0dc2aa2fee 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java @@ -70,7 +70,7 @@ public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsOb break; } default: - throw new IllegalArgumentException("Unknown stats type " + typeNew.toString()); + throw new IllegalArgumentException("Unknown stats type " + statsObjNew.getStatsData().getSetField()); } return agg; } diff --git ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java index f43992c85d..eb94b939f1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java @@ -18,11 +18,12 @@ package org.apache.hadoop.hive.ql; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.NodeUtils; import org.apache.hadoop.hive.ql.exec.NodeUtils.Function; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.BasicStatsTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskRunner; import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; @@ -64,7 +65,7 @@ private Context ctx; private boolean shutdown; - final Map statsTasks = new HashMap(1); + final Map statsTasks = new HashMap<>(1); public DriverContext() { } @@ -191,7 +192,9 @@ public void prepare(QueryPlan plan) { NodeUtils.iterateTask(rootTasks, StatsTask.class, new Function() { @Override public void apply(StatsTask statsTask) { - statsTasks.put(statsTask.getWork().getAggKey(), statsTask); + if(statsTask.getWork().getBasicStatsWork()!=null) { + statsTasks.put(statsTask.getWork().getBasicStatsWork().getAggKey(), statsTask); + } } }); } @@ -221,7 +224,7 @@ public void apply(FileSinkOperator fsOp) { } }); for (String statKey : statKeys) { - statsTasks.get(statKey).getWork().setSourceTask(mapredTask); + statsTasks.get(statKey).getWork().getBasicStatsWork().setSourceTask(mapredTask); } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/BasicStatsNoJobTask.java similarity index 98% rename from ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java rename to ql/src/java/org/apache/hadoop/hive/ql/exec/BasicStatsNoJobTask.java index 9c3a664b9a..beb68f2703 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/BasicStatsNoJobTask.java @@ -48,7 +48,7 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; @@ -71,16 +71,16 @@ * rows. This task can be used for computing basic stats like numFiles, numRows, fileSize, * rawDataSize from ORC footer. **/ -public class StatsNoJobTask extends Task implements Serializable { +public class BasicStatsNoJobTask extends Task implements Serializable { private static final long serialVersionUID = 1L; - private static transient final Logger LOG = LoggerFactory.getLogger(StatsNoJobTask.class); + private static transient final Logger LOG = LoggerFactory.getLogger(BasicStatsNoJobTask.class); private ConcurrentMap partUpdates; private Table table; private String tableFullName; private JobConf jc = null; - public StatsNoJobTask() { + public BasicStatsNoJobTask() { super(); } @@ -141,7 +141,6 @@ public void run() { // get the list of partitions org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); Map parameters = tPart.getParameters(); - try { Path dir = new Path(tPart.getSd().getLocation()); long numRows = 0; @@ -174,6 +173,7 @@ public void run() { } if (statsAvailable) { + parameters.put(StatsSetupConst.ROW_COUNT, String.valueOf(numRows)); parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(rawDataSize)); parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(fileSize)); @@ -280,7 +280,6 @@ private int aggregateStats(ExecutorService threadPool, Hive db) { parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(numFiles)); EnvironmentContext environmentContext = new EnvironmentContext(); environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); - db.alterTable(tableFullName, new Table(tTable), environmentContext); String msg = "Table " + tableFullName + " stats: [" + toString(parameters) + ']'; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/BasicStatsTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/BasicStatsTask.java new file mode 100644 index 0000000000..d3119040bd --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/BasicStatsTask.java @@ -0,0 +1,519 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hive.ql.exec; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; +import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; +import org.apache.hadoop.hive.ql.plan.LoadTableDesc; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.stats.StatsAggregator; +import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; +import org.apache.hadoop.hive.ql.stats.StatsFactory; +import org.apache.hadoop.hive.ql.stats.StatsPublisher; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * StatsTask implementation. StatsTask mainly deals with "collectable" stats. These are + * stats that require data scanning and are collected during query execution (unless the user + * explicitly requests data scanning just for the purpose of stats computation using the "ANALYZE" + * command. All other stats are computed directly by the MetaStore. The rationale being that the + * MetaStore layer covers all Thrift calls and provides better guarantees about the accuracy of + * those stats. + **/ +public class BasicStatsTask extends Task implements Serializable { + + private static final long serialVersionUID = 1L; + private static transient final Logger LOG = LoggerFactory.getLogger(BasicStatsTask.class); + + private Table table; + private Collection dpPartSpecs; + + public BasicStatsTask() { + super(); + dpPartSpecs = null; + } + + @Override + public int execute(DriverContext driverContext) { + if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { + return 0; + } + LOG.info("Executing stats task"); + // Make sure that it is either an ANALYZE, INSERT OVERWRITE (maybe load) or CTAS command + short workComponentsPresent = 0; + if (work.getLoadTableDesc() != null) { + workComponentsPresent++; + } + if (work.getTableSpecs() != null) { + workComponentsPresent++; + } + if (work.getLoadFileDesc() != null) { + workComponentsPresent++; + } + + assert (workComponentsPresent == 1); + + String tableName = ""; + Hive hive = getHive(); + try { + if (work.getLoadTableDesc() != null) { + tableName = work.getLoadTableDesc().getTable().getTableName(); + } else if (work.getTableSpecs() != null){ + tableName = work.getTableSpecs().tableName; + } else { + tableName = work.getLoadFileDesc().getDestinationCreateTable(); + } + + table = hive.getTable(tableName); + + } catch (HiveException e) { + LOG.error("Cannot get table " + tableName, e); + console.printError("Cannot get table " + tableName, e.toString()); + } + + return aggregateStats(hive); + + } + + @Override + public StageType getType() { + return StageType.STATS; + } + + @Override + public String getName() { + return "STATS"; + } + + private int aggregateStats(Hive db) { + + StatsAggregator statsAggregator = null; + int ret = 0; + StatsCollectionContext scc = null; + EnvironmentContext environmentContext = null; + try { + // Stats setup: + final Warehouse wh = new Warehouse(conf); + if (!getWork().getNoStatsAggregator() && !getWork().isNoScanAnalyzeCommand()) { + try { + scc = getContext(); + statsAggregator = createStatsAggregator(scc, conf); + } catch (HiveException e) { + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { + throw e; + } + console.printError(ErrorMsg.STATS_SKIPPING_BY_ERROR.getErrorCodedMsg(e.toString())); + } + } + + List partitions = getPartitionsList(db); + boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); + + String tableFullName = table.getDbName() + "." + table.getTableName(); + + if (partitions == null) { + org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable(); + Map parameters = tTable.getParameters(); + // In the following scenarios, we need to reset the stats to true. + // work.getTableSpecs() != null means analyze command + // work.getLoadTableDesc().getReplace() is true means insert overwrite command + // work.getLoadFileDesc().getDestinationCreateTable().isEmpty() means CTAS etc. + // acidTable will not have accurate stats unless it is set through analyze command. + if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); + } else if (work.getTableSpecs() != null + || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) + || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() + .getDestinationCreateTable().isEmpty())) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); + } + // work.getTableSpecs() == null means it is not analyze command + // and then if it is not followed by column stats, we should clean + // column stats + if (work.getTableSpecs() == null && !work.isFollowedByColStats()) { + StatsSetupConst.clearColumnStatsState(parameters); + } + // non-partitioned tables: + if (!existStats(parameters) && atomic) { + return 0; + } + + // The collectable stats for the aggregator needs to be cleared. + // For eg. if a file is being loaded, the old number of rows are not valid + if (work.isClearAggregatorStats()) { + // we choose to keep the invalid stats and only change the setting. + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); + } + + updateQuickStats(wh, parameters, tTable.getSd()); + if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { + if (statsAggregator != null) { + String prefix = getAggregationPrefix(table, null); + updateStats(statsAggregator, parameters, prefix, atomic); + } + // write table stats to metastore + if (!getWork().getNoStatsAggregator()) { + environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, + StatsSetupConst.TASK); + } + } + + getHive().alterTable(tableFullName, new Table(tTable), environmentContext); + if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { + console.printInfo("Table " + tableFullName + " stats: [" + toString(parameters) + ']'); + } + LOG.info("Table " + tableFullName + " stats: [" + toString(parameters) + ']'); + } else { + // Partitioned table: + // Need to get the old stats of the partition + // and update the table stats based on the old and new stats. + List updates = new ArrayList(); + + //Get the file status up-front for all partitions. Beneficial in cases of blob storage systems + final Map fileStatusMap = new ConcurrentHashMap(); + int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 1); + // In case thread count is set to 0, use single thread. + poolSize = Math.max(poolSize, 1); + final ExecutorService pool = Executors.newFixedThreadPool(poolSize, + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("stats-updater-thread-%d") + .build()); + final List> futures = Lists.newLinkedList(); + LOG.debug("Getting file stats of all partitions. threadpool size:" + poolSize); + try { + for(final Partition partn : partitions) { + final String partitionName = partn.getName(); + final org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); + Map parameters = tPart.getParameters(); + + if (!existStats(parameters) && atomic) { + continue; + } + futures.add(pool.submit(new Callable() { + @Override + public Void call() throws Exception { + FileStatus[] partfileStatus = wh.getFileStatusesForSD(tPart.getSd()); + fileStatusMap.put(partitionName, partfileStatus); + return null; + } + })); + } + pool.shutdown(); + for(Future future : futures) { + future.get(); + } + } catch (InterruptedException e) { + LOG.debug("Cancelling " + futures.size() + " file stats lookup tasks"); + //cancel other futures + for (Future future : futures) { + future.cancel(true); + } + // Fail the query if the stats are supposed to be reliable + if (work.isStatsReliable()) { + ret = 1; + } + } finally { + if (pool != null) { + pool.shutdownNow(); + } + LOG.debug("Finished getting file stats of all partitions"); + } + + for (Partition partn : partitions) { + // + // get the old partition stats + // + org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); + Map parameters = tPart.getParameters(); + if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); + } else if (work.getTableSpecs() != null + || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) + || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() + .getDestinationCreateTable().isEmpty())) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); + } + // work.getTableSpecs() == null means it is not analyze command + // and then if it is not followed by column stats, we should clean + // column stats + if (work.getTableSpecs() == null && !work.isFollowedByColStats()) { + StatsSetupConst.clearColumnStatsState(parameters); + } + //only when the stats exist, it is added to fileStatusMap + if (!fileStatusMap.containsKey(partn.getName())) { + continue; + } + + // The collectable stats for the aggregator needs to be cleared. + // For eg. if a file is being loaded, the old number of rows are not valid + if (work.isClearAggregatorStats()) { + // we choose to keep the invalid stats and only change the setting. + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); + } + + updateQuickStats(parameters, fileStatusMap.get(partn.getName())); + if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { + if (statsAggregator != null) { + String prefix = getAggregationPrefix(table, partn); + updateStats(statsAggregator, parameters, prefix, atomic); + } + if (!getWork().getNoStatsAggregator()) { + environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, + StatsSetupConst.TASK); + } + } + updates.add(new Partition(table, tPart)); + + if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { + console.printInfo("Partition " + tableFullName + partn.getSpec() + + " stats: [" + toString(parameters) + ']'); + } + LOG.info("Partition " + tableFullName + partn.getSpec() + + " stats: [" + toString(parameters) + ']'); + } + if (!updates.isEmpty()) { + db.alterPartitions(tableFullName, updates, environmentContext); + } + } + + } catch (Exception e) { + console.printInfo("[Warning] could not update stats.", + "Failed with exception " + e.getMessage() + "\n" + + StringUtils.stringifyException(e)); + + // Fail the query if the stats are supposed to be reliable + if (work.isStatsReliable()) { + ret = 1; + } + } finally { + if (statsAggregator != null) { + statsAggregator.closeConnection(scc); + } + } + // The return value of 0 indicates success, + // anything else indicates failure + return ret; + } + + private String getAggregationPrefix(Table table, Partition partition) + throws MetaException { + + // prefix is of the form dbName.tblName + String prefix = table.getDbName() + "." + MetaStoreUtils.encodeTableName(table.getTableName()); + if (partition != null) { + return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); + } + return prefix; + } + + private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { + String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); + StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); + if (factory == null) { + throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); + } + // initialize stats publishing table for noscan which has only stats task + // the rest of MR task following stats task initializes it in ExecDriver.java + StatsPublisher statsPublisher = factory.getStatsPublisher(); + if (!statsPublisher.init(scc)) { // creating stats table if not exists + throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); + } + + // manufacture a StatsAggregator + StatsAggregator statsAggregator = factory.getStatsAggregator(); + if (!statsAggregator.connect(scc)) { + throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); + } + return statsAggregator; + } + + private StatsCollectionContext getContext() throws HiveException { + + StatsCollectionContext scc = new StatsCollectionContext(conf); + Task sourceTask = getWork().getSourceTask(); + if (sourceTask == null) { + throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); + } + scc.setTask(sourceTask); + scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); + return scc; + } + + private boolean existStats(Map parameters) { + return parameters.containsKey(StatsSetupConst.ROW_COUNT) + || parameters.containsKey(StatsSetupConst.NUM_FILES) + || parameters.containsKey(StatsSetupConst.TOTAL_SIZE) + || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE) + || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS); + } + + private void updateStats(StatsAggregator statsAggregator, + Map parameters, String prefix, boolean atomic) + throws HiveException { + + String aggKey = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR; + + for (String statType : StatsSetupConst.statsRequireCompute) { + String value = statsAggregator.aggregateStats(aggKey, statType); + if (value != null && !value.isEmpty()) { + long longValue = Long.parseLong(value); + + if (work.getLoadTableDesc() != null && + !work.getLoadTableDesc().getReplace()) { + String originalValue = parameters.get(statType); + if (originalValue != null) { + longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid + } + } + parameters.put(statType, String.valueOf(longValue)); + } else { + if (atomic) { + throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); + } + } + } + } + + private void updateQuickStats(Warehouse wh, Map parameters, + StorageDescriptor desc) throws MetaException { + /** + * calculate fast statistics + */ + FileStatus[] partfileStatus = wh.getFileStatusesForSD(desc); + updateQuickStats(parameters, partfileStatus); + } + + private void updateQuickStats(Map parameters, + FileStatus[] partfileStatus) throws MetaException { + MetaStoreUtils.populateQuickStats(partfileStatus, parameters); + } + + private String toString(Map parameters) { + StringBuilder builder = new StringBuilder(); + for (String statType : StatsSetupConst.supportedStats) { + String value = parameters.get(statType); + if (value != null) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(statType).append('=').append(value); + } + } + return builder.toString(); + } + + /** + * Get the list of partitions that need to update statistics. + * TODO: we should reuse the Partitions generated at compile time + * since getting the list of partitions is quite expensive. + * + * @return a list of partitions that need to update statistics. + * @throws HiveException + */ + private List getPartitionsList(Hive db) throws HiveException { + if (work.getLoadFileDesc() != null) { + return null; //we are in CTAS, so we know there are no partitions + } + + List list = new ArrayList(); + + if (work.getTableSpecs() != null) { + + // ANALYZE command + TableSpec tblSpec = work.getTableSpecs(); + table = tblSpec.tableHandle; + if (!table.isPartitioned()) { + return null; + } + // get all partitions that matches with the partition spec + List partitions = tblSpec.partitions; + if (partitions != null) { + for (Partition partn : partitions) { + list.add(partn); + } + } + } else if (work.getLoadTableDesc() != null) { + + // INSERT OVERWRITE command + LoadTableDesc tbd = work.getLoadTableDesc(); + table = db.getTable(tbd.getTable().getTableName()); + if (!table.isPartitioned()) { + return null; + } + DynamicPartitionCtx dpCtx = tbd.getDPCtx(); + if (dpCtx != null && dpCtx.getNumDPCols() > 0) { // dynamic partitions + // If no dynamic partitions are generated, dpPartSpecs may not be initialized + if (dpPartSpecs != null) { + // load the list of DP partitions and return the list of partition specs + list.addAll(dpPartSpecs); + } + } else { // static partition + Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); + list.add(partn); + } + } + return list; + } + + public Collection getDpPartSpecs() { + return dpPartSpecs; + } + + public void setDpPartSpecs(Collection dpPartSpecs) { + this.dpPartSpecs = dpPartSpecs; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java deleted file mode 100644 index 2b2c004fea..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java +++ /dev/null @@ -1,452 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import java.io.IOException; -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; -import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Date; -import org.apache.hadoop.hive.metastore.api.Decimal; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; -import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; -import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.QueryPlan; -import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; -import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * ColumnStatsTask implementation. - **/ - -public class ColumnStatsTask extends Task implements Serializable { - private static final long serialVersionUID = 1L; - private FetchOperator ftOp; - private static transient final Logger LOG = LoggerFactory.getLogger(ColumnStatsTask.class); - - public ColumnStatsTask() { - super(); - } - - @Override - public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx, - CompilationOpContext opContext) { - super.initialize(queryState, queryPlan, ctx, opContext); - work.initializeForFetch(opContext); - try { - JobConf job = new JobConf(conf); - ftOp = new FetchOperator(work.getfWork(), job); - } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - throw new RuntimeException(e); - } - } - - private void unpackBooleanStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { - long v = ((LongObjectInspector) oi).get(o); - if (fName.equals("counttrues")) { - statsObj.getStatsData().getBooleanStats().setNumTrues(v); - } else if (fName.equals("countfalses")) { - statsObj.getStatsData().getBooleanStats().setNumFalses(v); - } else if (fName.equals("countnulls")) { - statsObj.getStatsData().getBooleanStats().setNumNulls(v); - } - } - - @SuppressWarnings("serial") - class UnsupportedDoubleException extends Exception { - } - - private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { - if (fName.equals("countnulls")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getDoubleStats().setNumNulls(v); - } else if (fName.equals("numdistinctvalues")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getDoubleStats().setNumDVs(v); - } else if (fName.equals("max")) { - double d = ((DoubleObjectInspector) oi).get(o); - if (Double.isInfinite(d) || Double.isNaN(d)) { - throw new UnsupportedDoubleException(); - } - statsObj.getStatsData().getDoubleStats().setHighValue(d); - } else if (fName.equals("min")) { - double d = ((DoubleObjectInspector) oi).get(o); - if (Double.isInfinite(d) || Double.isNaN(d)) { - throw new UnsupportedDoubleException(); - } - statsObj.getStatsData().getDoubleStats().setLowValue(d); - } else if (fName.equals("ndvbitvector")) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getDoubleStats().setBitVectors(buf); - } - } - - private void unpackDecimalStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { - if (fName.equals("countnulls")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getDecimalStats().setNumNulls(v); - } else if (fName.equals("numdistinctvalues")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getDecimalStats().setNumDVs(v); - } else if (fName.equals("max")) { - HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getDecimalStats().setHighValue(convertToThriftDecimal(d)); - } else if (fName.equals("min")) { - HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getDecimalStats().setLowValue(convertToThriftDecimal(d)); - } else if (fName.equals("ndvbitvector")) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getDecimalStats().setBitVectors(buf); - } - } - - private Decimal convertToThriftDecimal(HiveDecimal d) { - return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), (short)d.scale()); - } - - private void unpackLongStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { - if (fName.equals("countnulls")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getLongStats().setNumNulls(v); - } else if (fName.equals("numdistinctvalues")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getLongStats().setNumDVs(v); - } else if (fName.equals("max")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getLongStats().setHighValue(v); - } else if (fName.equals("min")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getLongStats().setLowValue(v); - } else if (fName.equals("ndvbitvector")) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getLongStats().setBitVectors(buf); - } - } - - private void unpackStringStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { - if (fName.equals("countnulls")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getStringStats().setNumNulls(v); - } else if (fName.equals("numdistinctvalues")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getStringStats().setNumDVs(v); - } else if (fName.equals("avglength")) { - double d = ((DoubleObjectInspector) oi).get(o); - statsObj.getStatsData().getStringStats().setAvgColLen(d); - } else if (fName.equals("maxlength")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getStringStats().setMaxColLen(v); - } else if (fName.equals("ndvbitvector")) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getStringStats().setBitVectors(buf); - } - } - - private void unpackBinaryStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { - if (fName.equals("countnulls")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getBinaryStats().setNumNulls(v); - } else if (fName.equals("avglength")) { - double d = ((DoubleObjectInspector) oi).get(o); - statsObj.getStatsData().getBinaryStats().setAvgColLen(d); - } else if (fName.equals("maxlength")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getBinaryStats().setMaxColLen(v); - } - } - - private void unpackDateStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { - if (fName.equals("countnulls")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getDateStats().setNumNulls(v); - } else if (fName.equals("numdistinctvalues")) { - long v = ((LongObjectInspector) oi).get(o); - statsObj.getStatsData().getDateStats().setNumDVs(v); - } else if (fName.equals("max")) { - DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); - statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); - } else if (fName.equals("min")) { - DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); - statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); - } else if (fName.equals("ndvbitvector")) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); - statsObj.getStatsData().getDateStats().setBitVectors(buf); - } - } - - private void unpackPrimitiveObject (ObjectInspector oi, Object o, String fieldName, - ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { - if (o == null) { - return; - } - // First infer the type of object - if (fieldName.equals("columntype")) { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; - String s = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); - ColumnStatisticsData statsData = new ColumnStatisticsData(); - - if (s.equalsIgnoreCase("long")) { - LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); - statsData.setLongStats(longStats); - statsObj.setStatsData(statsData); - } else if (s.equalsIgnoreCase("double")) { - DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector(); - statsData.setDoubleStats(doubleStats); - statsObj.setStatsData(statsData); - } else if (s.equalsIgnoreCase("string")) { - StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); - statsData.setStringStats(stringStats); - statsObj.setStatsData(statsData); - } else if (s.equalsIgnoreCase("boolean")) { - BooleanColumnStatsData booleanStats = new BooleanColumnStatsData(); - statsData.setBooleanStats(booleanStats); - statsObj.setStatsData(statsData); - } else if (s.equalsIgnoreCase("binary")) { - BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); - statsData.setBinaryStats(binaryStats); - statsObj.setStatsData(statsData); - } else if (s.equalsIgnoreCase("decimal")) { - DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector(); - statsData.setDecimalStats(decimalStats); - statsObj.setStatsData(statsData); - } else if (s.equalsIgnoreCase("date")) { - DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector(); - statsData.setDateStats(dateStats); - statsObj.setStatsData(statsData); - } - } else { - // invoke the right unpack method depending on data type of the column - if (statsObj.getStatsData().isSetBooleanStats()) { - unpackBooleanStats(oi, o, fieldName, statsObj); - } else if (statsObj.getStatsData().isSetLongStats()) { - unpackLongStats(oi, o, fieldName, statsObj); - } else if (statsObj.getStatsData().isSetDoubleStats()) { - unpackDoubleStats(oi,o,fieldName, statsObj); - } else if (statsObj.getStatsData().isSetStringStats()) { - unpackStringStats(oi, o, fieldName, statsObj); - } else if (statsObj.getStatsData().isSetBinaryStats()) { - unpackBinaryStats(oi, o, fieldName, statsObj); - } else if (statsObj.getStatsData().isSetDecimalStats()) { - unpackDecimalStats(oi, o, fieldName, statsObj); - } else if (statsObj.getStatsData().isSetDateStats()) { - unpackDateStats(oi, o, fieldName, statsObj); - } - } - } - - private void unpackStructObject(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { - if (oi.getCategory() != ObjectInspector.Category.STRUCT) { - throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString()); - } - - StructObjectInspector soi = (StructObjectInspector) oi; - List fields = soi.getAllStructFieldRefs(); - List list = soi.getStructFieldsDataAsList(o); - - for (int i = 0; i < fields.size(); i++) { - // Get the field objectInspector, fieldName and the field object. - ObjectInspector foi = fields.get(i).getFieldObjectInspector(); - Object f = (list == null ? null : list.get(i)); - String fieldName = fields.get(i).getFieldName(); - - if (foi.getCategory() == ObjectInspector.Category.PRIMITIVE) { - unpackPrimitiveObject(foi, f, fieldName, cStatsObj); - } else { - unpackStructObject(foi, f, fieldName, cStatsObj); - } - } - } - - private List constructColumnStatsFromPackedRows( - Hive db) throws HiveException, MetaException, IOException { - - String currentDb = SessionState.get().getCurrentDatabase(); - String tableName = work.getColStats().getTableName(); - String partName = null; - List colName = work.getColStats().getColName(); - List colType = work.getColStats().getColType(); - boolean isTblLevel = work.getColStats().isTblLevel(); - - List stats = new ArrayList(); - InspectableObject packedRow; - Table tbl = db.getTable(currentDb, tableName); - while ((packedRow = ftOp.getNextRow()) != null) { - if (packedRow.oi.getCategory() != ObjectInspector.Category.STRUCT) { - throw new HiveException("Unexpected object type encountered while unpacking row"); - } - - List statsObjs = new ArrayList(); - StructObjectInspector soi = (StructObjectInspector) packedRow.oi; - List fields = soi.getAllStructFieldRefs(); - List list = soi.getStructFieldsDataAsList(packedRow.o); - - List partColSchema = tbl.getPartCols(); - // Partition columns are appended at end, we only care about stats column - int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size(); - for (int i = 0; i < numOfStatCols; i++) { - // Get the field objectInspector, fieldName and the field object. - ObjectInspector foi = fields.get(i).getFieldObjectInspector(); - Object f = (list == null ? null : list.get(i)); - String fieldName = fields.get(i).getFieldName(); - ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); - statsObj.setColName(colName.get(i)); - statsObj.setColType(colType.get(i)); - try { - unpackStructObject(foi, f, fieldName, statsObj); - statsObjs.add(statsObj); - } catch (UnsupportedDoubleException e) { - // due to infinity or nan. - LOG.info("Because " + colName.get(i) + " is infinite or NaN, we skip stats."); - } - } - - if (!isTblLevel) { - List partVals = new ArrayList(); - // Iterate over partition columns to figure out partition name - for (int i = fields.size() - partColSchema.size(); i < fields.size(); i++) { - Object partVal = ((PrimitiveObjectInspector)fields.get(i).getFieldObjectInspector()). - getPrimitiveJavaObject(list.get(i)); - partVals.add(partVal == null ? // could be null for default partition - this.conf.getVar(ConfVars.DEFAULTPARTITIONNAME) : partVal.toString()); - } - partName = Warehouse.makePartName(partColSchema, partVals); - } - String [] names = Utilities.getDbTableName(currentDb, tableName); - ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, isTblLevel); - ColumnStatistics colStats = new ColumnStatistics(); - colStats.setStatsDesc(statsDesc); - colStats.setStatsObj(statsObjs); - if (!statsObjs.isEmpty()) { - stats.add(colStats); - } - } - ftOp.clearFetchContext(); - return stats; - } - - private ColumnStatisticsDesc getColumnStatsDesc(String dbName, String tableName, - String partName, boolean isTblLevel) - { - ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); - statsDesc.setDbName(dbName); - statsDesc.setTableName(tableName); - statsDesc.setIsTblLevel(isTblLevel); - - if (!isTblLevel) { - statsDesc.setPartName(partName); - } else { - statsDesc.setPartName(null); - } - return statsDesc; - } - - private int persistColumnStats(Hive db) throws HiveException, MetaException, IOException { - // Construct a column statistics object from the result - List colStats = constructColumnStatsFromPackedRows(db); - // Persist the column statistics object to the metastore - // Note, this function is shared for both table and partition column stats. - if (colStats.isEmpty()) { - return 0; - } - SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); - if (work.getColStats() != null && work.getColStats().getNumBitVector() > 0) { - request.setNeedMerge(true); - } - db.setPartitionColumnStatistics(request); - return 0; - } - - @Override - public int execute(DriverContext driverContext) { - if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { - return 0; - } - try { - Hive db = getHive(); - return persistColumnStats(db); - } catch (Exception e) { - LOG.error("Failed to run column stats task", e); - } - return 1; - } - - @Override - public StageType getType() { - return StageType.COLUMNSTATS; - } - - @Override - public String getName() { - return "COLUMNSTATS TASK"; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index c22d69bb19..120ed52563 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -16,493 +16,483 @@ * limitations under the License. */ - package org.apache.hadoop.hive.ql.exec; +import java.io.IOException; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; +import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Date; +import org.apache.hadoop.hive.metastore.api.Decimal; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; +import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; -import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; -import org.apache.hadoop.hive.ql.plan.LoadTableDesc; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.hive.ql.stats.StatsAggregator; -import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; -import org.apache.hadoop.hive.ql.stats.StatsFactory; -import org.apache.hadoop.hive.ql.stats.StatsPublisher; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; - -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * StatsTask implementation. StatsTask mainly deals with "collectable" stats. These are - * stats that require data scanning and are collected during query execution (unless the user - * explicitly requests data scanning just for the purpose of stats computation using the "ANALYZE" - * command. All other stats are computed directly by the MetaStore. The rationale being that the - * MetaStore layer covers all Thrift calls and provides better guarantees about the accuracy of - * those stats. + * StatsTask implementation. **/ -public class StatsTask extends Task implements Serializable { +public class StatsTask extends Task implements Serializable { private static final long serialVersionUID = 1L; + private FetchOperator ftOp; private static transient final Logger LOG = LoggerFactory.getLogger(StatsTask.class); - private Table table; - private Collection dpPartSpecs; - public StatsTask() { super(); - dpPartSpecs = null; } @Override - protected void receiveFeed(FeedType feedType, Object feedValue) { - // this method should be called by MoveTask when there are dynamic partitions generated - if (feedType == FeedType.DYNAMIC_PARTITIONS) { - dpPartSpecs = (Collection) feedValue; + public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx, + CompilationOpContext opContext) { + super.initialize(queryState, queryPlan, ctx, opContext); + if (work.getfWork() != null) { + work.initializeForFetch(opContext); + try { + JobConf job = new JobConf(conf); + ftOp = new FetchOperator(work.getfWork(), job); + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + throw new RuntimeException(e); + } } } - @Override - public int execute(DriverContext driverContext) { - if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { - return 0; - } - LOG.info("Executing stats task"); - // Make sure that it is either an ANALYZE, INSERT OVERWRITE (maybe load) or CTAS command - short workComponentsPresent = 0; - if (work.getLoadTableDesc() != null) { - workComponentsPresent++; - } - if (work.getTableSpecs() != null) { - workComponentsPresent++; - } - if (work.getLoadFileDesc() != null) { - workComponentsPresent++; + private void unpackBooleanStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) { + long v = ((LongObjectInspector) oi).get(o); + if (fName.equals("counttrues")) { + statsObj.getStatsData().getBooleanStats().setNumTrues(v); + } else if (fName.equals("countfalses")) { + statsObj.getStatsData().getBooleanStats().setNumFalses(v); + } else if (fName.equals("countnulls")) { + statsObj.getStatsData().getBooleanStats().setNumNulls(v); } + } - assert (workComponentsPresent == 1); + @SuppressWarnings("serial") + class UnsupportedDoubleException extends Exception { + } - String tableName = ""; - Hive hive = getHive(); - try { - if (work.getLoadTableDesc() != null) { - tableName = work.getLoadTableDesc().getTable().getTableName(); - } else if (work.getTableSpecs() != null){ - tableName = work.getTableSpecs().tableName; - } else { - tableName = work.getLoadFileDesc().getDestinationCreateTable(); + private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { + if (fName.equals("countnulls")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getDoubleStats().setNumNulls(v); + } else if (fName.equals("numdistinctvalues")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getDoubleStats().setNumDVs(v); + } else if (fName.equals("max")) { + double d = ((DoubleObjectInspector) oi).get(o); + if (Double.isInfinite(d) || Double.isNaN(d)) { + throw new UnsupportedDoubleException(); } + statsObj.getStatsData().getDoubleStats().setHighValue(d); + } else if (fName.equals("min")) { + double d = ((DoubleObjectInspector) oi).get(o); + if (Double.isInfinite(d) || Double.isNaN(d)) { + throw new UnsupportedDoubleException(); + } + statsObj.getStatsData().getDoubleStats().setLowValue(d); + } else if (fName.equals("ndvbitvector")) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getDoubleStats().setBitVectors(buf); + ; + } + } - table = hive.getTable(tableName); - - } catch (HiveException e) { - LOG.error("Cannot get table " + tableName, e); - console.printError("Cannot get table " + tableName, e.toString()); + private void unpackDecimalStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) { + if (fName.equals("countnulls")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getDecimalStats().setNumNulls(v); + } else if (fName.equals("numdistinctvalues")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getDecimalStats().setNumDVs(v); + } else if (fName.equals("max")) { + HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getDecimalStats().setHighValue(convertToThriftDecimal(d)); + } else if (fName.equals("min")) { + HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getDecimalStats().setLowValue(convertToThriftDecimal(d)); + } else if (fName.equals("ndvbitvector")) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getDecimalStats().setBitVectors(buf); + ; } + } - return aggregateStats(hive); + private Decimal convertToThriftDecimal(HiveDecimal d) { + return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), (short) d.scale()); + } + private void unpackLongStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) { + if (fName.equals("countnulls")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getLongStats().setNumNulls(v); + } else if (fName.equals("numdistinctvalues")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getLongStats().setNumDVs(v); + } else if (fName.equals("max")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getLongStats().setHighValue(v); + } else if (fName.equals("min")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getLongStats().setLowValue(v); + } else if (fName.equals("ndvbitvector")) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getLongStats().setBitVectors(buf); + ; + } } - @Override - public StageType getType() { - return StageType.STATS; + private void unpackStringStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) { + if (fName.equals("countnulls")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getStringStats().setNumNulls(v); + } else if (fName.equals("numdistinctvalues")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getStringStats().setNumDVs(v); + } else if (fName.equals("avglength")) { + double d = ((DoubleObjectInspector) oi).get(o); + statsObj.getStatsData().getStringStats().setAvgColLen(d); + } else if (fName.equals("maxlength")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getStringStats().setMaxColLen(v); + } else if (fName.equals("ndvbitvector")) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getStringStats().setBitVectors(buf); + ; + } } - @Override - public String getName() { - return "STATS"; + private void unpackBinaryStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) { + if (fName.equals("countnulls")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getBinaryStats().setNumNulls(v); + } else if (fName.equals("avglength")) { + double d = ((DoubleObjectInspector) oi).get(o); + statsObj.getStatsData().getBinaryStats().setAvgColLen(d); + } else if (fName.equals("maxlength")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getBinaryStats().setMaxColLen(v); + } } - private int aggregateStats(Hive db) { + private void unpackDateStats(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj statsObj) { + if (fName.equals("countnulls")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getDateStats().setNumNulls(v); + } else if (fName.equals("numdistinctvalues")) { + long v = ((LongObjectInspector) oi).get(o); + statsObj.getStatsData().getDateStats().setNumDVs(v); + } else if (fName.equals("max")) { + DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); + statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); + } else if (fName.equals("min")) { + DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); + statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); + } else if (fName.equals("ndvbitvector")) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); + statsObj.getStatsData().getDateStats().setBitVectors(buf); + ; + } + } - StatsAggregator statsAggregator = null; - int ret = 0; - StatsCollectionContext scc = null; - EnvironmentContext environmentContext = null; - try { - // Stats setup: - final Warehouse wh = new Warehouse(conf); - if (!getWork().getNoStatsAggregator() && !getWork().isNoScanAnalyzeCommand()) { - try { - scc = getContext(); - statsAggregator = createStatsAggregator(scc, conf); - } catch (HiveException e) { - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { - throw e; - } - console.printError(ErrorMsg.STATS_SKIPPING_BY_ERROR.getErrorCodedMsg(e.toString())); - } + private void unpackPrimitiveObject(ObjectInspector oi, Object o, String fieldName, + ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { + if (o == null) { + return; + } + // First infer the type of object + if (fieldName.equals("columntype")) { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + String s = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); + ColumnStatisticsData statsData = new ColumnStatisticsData(); + + if (s.equalsIgnoreCase("long")) { + LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); + statsData.setLongStats(longStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("double")) { + DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector(); + statsData.setDoubleStats(doubleStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("string")) { + StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); + statsData.setStringStats(stringStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("boolean")) { + BooleanColumnStatsData booleanStats = new BooleanColumnStatsData(); + statsData.setBooleanStats(booleanStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("binary")) { + BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); + statsData.setBinaryStats(binaryStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("decimal")) { + DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector(); + statsData.setDecimalStats(decimalStats); + statsObj.setStatsData(statsData); + } else if (s.equalsIgnoreCase("date")) { + DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector(); + statsData.setDateStats(dateStats); + statsObj.setStatsData(statsData); + } + } else { + // invoke the right unpack method depending on data type of the column + if (statsObj.getStatsData().isSetBooleanStats()) { + unpackBooleanStats(oi, o, fieldName, statsObj); + } else if (statsObj.getStatsData().isSetLongStats()) { + unpackLongStats(oi, o, fieldName, statsObj); + } else if (statsObj.getStatsData().isSetDoubleStats()) { + unpackDoubleStats(oi, o, fieldName, statsObj); + } else if (statsObj.getStatsData().isSetStringStats()) { + unpackStringStats(oi, o, fieldName, statsObj); + } else if (statsObj.getStatsData().isSetBinaryStats()) { + unpackBinaryStats(oi, o, fieldName, statsObj); + } else if (statsObj.getStatsData().isSetDecimalStats()) { + unpackDecimalStats(oi, o, fieldName, statsObj); + } else if (statsObj.getStatsData().isSetDateStats()) { + unpackDateStats(oi, o, fieldName, statsObj); } + } + } - List partitions = getPartitionsList(db); - boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); - - String tableFullName = table.getDbName() + "." + table.getTableName(); - - if (partitions == null) { - org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable(); - Map parameters = tTable.getParameters(); - // In the following scenarios, we need to reset the stats to true. - // work.getTableSpecs() != null means analyze command - // work.getLoadTableDesc().getReplace() is true means insert overwrite command - // work.getLoadFileDesc().getDestinationCreateTable().isEmpty() means CTAS etc. - // acidTable will not have accurate stats unless it is set through analyze command. - if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } else if (work.getTableSpecs() != null - || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) - || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() - .getDestinationCreateTable().isEmpty())) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); - } - // non-partitioned tables: - if (!existStats(parameters) && atomic) { - return 0; - } + private void unpackStructObject(ObjectInspector oi, Object o, String fName, + ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { + if (oi.getCategory() != ObjectInspector.Category.STRUCT) { + throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString()); + } - // The collectable stats for the aggregator needs to be cleared. - // For eg. if a file is being loaded, the old number of rows are not valid - if (work.isClearAggregatorStats()) { - // we choose to keep the invalid stats and only change the setting. - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + List list = soi.getStructFieldsDataAsList(o); - updateQuickStats(wh, parameters, tTable.getSd()); - if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { - if (statsAggregator != null) { - String prefix = getAggregationPrefix(table, null); - updateStats(statsAggregator, parameters, prefix, atomic); - } - // write table stats to metastore - if (!getWork().getNoStatsAggregator()) { - environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, - StatsSetupConst.TASK); - } - } + for (int i = 0; i < fields.size(); i++) { + // Get the field objectInspector, fieldName and the field object. + ObjectInspector foi = fields.get(i).getFieldObjectInspector(); + Object f = (list == null ? null : list.get(i)); + String fieldName = fields.get(i).getFieldName(); - getHive().alterTable(tableFullName, new Table(tTable), environmentContext); - if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { - console.printInfo("Table " + tableFullName + " stats: [" + toString(parameters) + ']'); - } - LOG.info("Table " + tableFullName + " stats: [" + toString(parameters) + ']'); + if (foi.getCategory() == ObjectInspector.Category.PRIMITIVE) { + unpackPrimitiveObject(foi, f, fieldName, cStatsObj); } else { - // Partitioned table: - // Need to get the old stats of the partition - // and update the table stats based on the old and new stats. - List updates = new ArrayList(); - - //Get the file status up-front for all partitions. Beneficial in cases of blob storage systems - final Map fileStatusMap = new ConcurrentHashMap(); - int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 1); - // In case thread count is set to 0, use single thread. - poolSize = Math.max(poolSize, 1); - final ExecutorService pool = Executors.newFixedThreadPool(poolSize, - new ThreadFactoryBuilder().setDaemon(true) - .setNameFormat("stats-updater-thread-%d") - .build()); - final List> futures = Lists.newLinkedList(); - LOG.debug("Getting file stats of all partitions. threadpool size:" + poolSize); - try { - for(final Partition partn : partitions) { - final String partitionName = partn.getName(); - final org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); - Map parameters = tPart.getParameters(); - - if (!existStats(parameters) && atomic) { - continue; - } - futures.add(pool.submit(new Callable() { - @Override - public Void call() throws Exception { - FileStatus[] partfileStatus = wh.getFileStatusesForSD(tPart.getSd()); - fileStatusMap.put(partitionName, partfileStatus); - return null; - } - })); - } - pool.shutdown(); - for(Future future : futures) { - future.get(); - } - } catch (InterruptedException e) { - LOG.debug("Cancelling " + futures.size() + " file stats lookup tasks"); - //cancel other futures - for (Future future : futures) { - future.cancel(true); - } - // Fail the query if the stats are supposed to be reliable - if (work.isStatsReliable()) { - ret = 1; - } - } finally { - if (pool != null) { - pool.shutdownNow(); - } - LOG.debug("Finished getting file stats of all partitions"); - } - - for (Partition partn : partitions) { - // - // get the old partition stats - // - org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); - Map parameters = tPart.getParameters(); - if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } else if (work.getTableSpecs() != null - || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) - || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() - .getDestinationCreateTable().isEmpty())) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); - } - //only when the stats exist, it is added to fileStatusMap - if (!fileStatusMap.containsKey(partn.getName())) { - continue; - } - - // The collectable stats for the aggregator needs to be cleared. - // For eg. if a file is being loaded, the old number of rows are not valid - if (work.isClearAggregatorStats()) { - // we choose to keep the invalid stats and only change the setting. - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } - - updateQuickStats(parameters, fileStatusMap.get(partn.getName())); - if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { - if (statsAggregator != null) { - String prefix = getAggregationPrefix(table, partn); - updateStats(statsAggregator, parameters, prefix, atomic); - } - if (!getWork().getNoStatsAggregator()) { - environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, - StatsSetupConst.TASK); - } - } - updates.add(new Partition(table, tPart)); - - if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { - console.printInfo("Partition " + tableFullName + partn.getSpec() + - " stats: [" + toString(parameters) + ']'); - } - LOG.info("Partition " + tableFullName + partn.getSpec() + - " stats: [" + toString(parameters) + ']'); - } - if (!updates.isEmpty()) { - db.alterPartitions(tableFullName, updates, environmentContext); - } + unpackStructObject(foi, f, fieldName, cStatsObj); } + } + } - } catch (Exception e) { - console.printInfo("[Warning] could not update stats.", - "Failed with exception " + e.getMessage() + "\n" - + StringUtils.stringifyException(e)); + private List constructColumnStatsFromPackedRows(Hive db) throws HiveException, + MetaException, IOException { + + String currentDb = SessionState.get().getCurrentDatabase(); + String tableName = work.getColStats().getTableName(); + String partName = null; + List colName = work.getColStats().getColName(); + List colType = work.getColStats().getColType(); + boolean isTblLevel = work.getColStats().isTblLevel(); + + List stats = new ArrayList(); + InspectableObject packedRow; + Table tbl = db.getTable(currentDb, tableName); + while ((packedRow = ftOp.getNextRow()) != null) { + if (packedRow.oi.getCategory() != ObjectInspector.Category.STRUCT) { + throw new HiveException("Unexpected object type encountered while unpacking row"); + } - // Fail the query if the stats are supposed to be reliable - if (work.isStatsReliable()) { - ret = 1; + List statsObjs = new ArrayList(); + StructObjectInspector soi = (StructObjectInspector) packedRow.oi; + List fields = soi.getAllStructFieldRefs(); + List list = soi.getStructFieldsDataAsList(packedRow.o); + + List partColSchema = tbl.getPartCols(); + // Partition columns are appended at end, we only care about stats column + int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size(); + for (int i = 0; i < numOfStatCols; i++) { + // Get the field objectInspector, fieldName and the field object. + ObjectInspector foi = fields.get(i).getFieldObjectInspector(); + Object f = (list == null ? null : list.get(i)); + String fieldName = fields.get(i).getFieldName(); + ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); + statsObj.setColName(colName.get(i)); + statsObj.setColType(colType.get(i)); + try { + unpackStructObject(foi, f, fieldName, statsObj); + statsObjs.add(statsObj); + } catch (UnsupportedDoubleException e) { + // due to infinity or nan. + LOG.info("Because " + colName.get(i) + " is infinite or NaN, we skip stats."); + } } - } finally { - if (statsAggregator != null) { - statsAggregator.closeConnection(scc); + + if (!isTblLevel) { + List partVals = new ArrayList(); + // Iterate over partition columns to figure out partition name + for (int i = fields.size() - partColSchema.size(); i < fields.size(); i++) { + Object partVal = ((PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector()) + .getPrimitiveJavaObject(list.get(i)); + partVals.add(partVal == null ? // could be null for default partition + this.conf.getVar(ConfVars.DEFAULTPARTITIONNAME) + : partVal.toString()); + } + partName = Warehouse.makePartName(partColSchema, partVals); + } + String[] names = Utilities.getDbTableName(currentDb, tableName); + ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, isTblLevel); + ColumnStatistics colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + if (!colStats.getStatsObj().isEmpty()) { + stats.add(colStats); } } - // The return value of 0 indicates success, - // anything else indicates failure - return ret; + ftOp.clearFetchContext(); + return stats; } - private String getAggregationPrefix(Table table, Partition partition) - throws MetaException { - - // prefix is of the form dbName.tblName - String prefix = table.getDbName() + "." + MetaStoreUtils.encodeTableName(table.getTableName()); - if (partition != null) { - return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); + private ColumnStatisticsDesc getColumnStatsDesc(String dbName, String tableName, String partName, + boolean isTblLevel) { + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tableName); + statsDesc.setIsTblLevel(isTblLevel); + + if (!isTblLevel) { + statsDesc.setPartName(partName); + } else { + statsDesc.setPartName(null); } - return prefix; + return statsDesc; } - private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { - String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); - StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); - if (factory == null) { - throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); - } - // initialize stats publishing table for noscan which has only stats task - // the rest of MR task following stats task initializes it in ExecDriver.java - StatsPublisher statsPublisher = factory.getStatsPublisher(); - if (!statsPublisher.init(scc)) { // creating stats table if not exists - throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); - } - - // manufacture a StatsAggregator - StatsAggregator statsAggregator = factory.getStatsAggregator(); - if (!statsAggregator.connect(scc)) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); + private int persistColumnStats(Hive db) throws HiveException, MetaException, IOException { + // Construct a column statistics object from the result + List colStats = constructColumnStatsFromPackedRows(db); + // Persist the column statistics object to the metastore + // Note, this function is shared for both table and partition column stats. + if (colStats.isEmpty()) { + return 0; } - return statsAggregator; + SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); + request.setNeedMerge(work.getColStats().isNeedMerge()); + db.setPartitionColumnStatistics(request); + return 0; } - private StatsCollectionContext getContext() throws HiveException { - - StatsCollectionContext scc = new StatsCollectionContext(conf); - Task sourceTask = getWork().getSourceTask(); - if (sourceTask == null) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); + @Override + public int execute(DriverContext driverContext) { + if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { + return 0; } - scc.setTask(sourceTask); - scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); - return scc; - } - - private boolean existStats(Map parameters) { - return parameters.containsKey(StatsSetupConst.ROW_COUNT) - || parameters.containsKey(StatsSetupConst.NUM_FILES) - || parameters.containsKey(StatsSetupConst.TOTAL_SIZE) - || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE) - || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS); - } - private void updateStats(StatsAggregator statsAggregator, - Map parameters, String prefix, boolean atomic) - throws HiveException { - - String aggKey = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR; - - for (String statType : StatsSetupConst.statsRequireCompute) { - String value = statsAggregator.aggregateStats(aggKey, statType); - if (value != null && !value.isEmpty()) { - long longValue = Long.parseLong(value); + // TODO: merge BasicStatsWork and BasicStatsNoJobWork + if (work.getBasicStatsWork() != null && work.getBasicStatsNoJobWork() != null) { + LOG.error("Can not have both basic stats work and stats no job work!"); + return 1; + } + int ret = 0; + if (work.getBasicStatsWork() != null) { + work.getBasicStatsWork().setFollowedByColStats(work.getfWork() != null); + BasicStatsTask basicStatsTask = (BasicStatsTask) TaskFactory.get(work.getBasicStatsWork(), conf); + basicStatsTask.initialize(queryState, queryPlan, driverContext, null); + basicStatsTask.setDpPartSpecs(dpPartSpecs); + ret = basicStatsTask.execute(driverContext); + } + if (work.getBasicStatsNoJobWork() != null) { + BasicStatsNoJobTask basicStatsTask = (BasicStatsNoJobTask) TaskFactory.get(work.getBasicStatsNoJobWork(), conf); + basicStatsTask.initialize(queryState, queryPlan, driverContext, null); + ret = basicStatsTask.execute(driverContext); + } + if (ret != 0) { + return ret; + } - if (work.getLoadTableDesc() != null && - !work.getLoadTableDesc().getReplace()) { - String originalValue = parameters.get(statType); - if (originalValue != null) { - longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid - } - } - parameters.put(statType, String.valueOf(longValue)); - } else { - if (atomic) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); - } + if (work.getfWork() != null) { + try { + Hive db = getHive(); + return persistColumnStats(db); + } catch (Exception e) { + LOG.error("Failed to run column stats task", e); + return 1; } } + return 0; } - private void updateQuickStats(Warehouse wh, Map parameters, - StorageDescriptor desc) throws MetaException { - /** - * calculate fast statistics - */ - FileStatus[] partfileStatus = wh.getFileStatusesForSD(desc); - updateQuickStats(parameters, partfileStatus); - } - - private void updateQuickStats(Map parameters, - FileStatus[] partfileStatus) throws MetaException { - MetaStoreUtils.populateQuickStats(partfileStatus, parameters); + @Override + public StageType getType() { + return StageType.COLUMNSTATS; } - private String toString(Map parameters) { - StringBuilder builder = new StringBuilder(); - for (String statType : StatsSetupConst.supportedStats) { - String value = parameters.get(statType); - if (value != null) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(statType).append('=').append(value); - } - } - return builder.toString(); + @Override + public String getName() { + return "COLUMNSTATS TASK"; } - /** - * Get the list of partitions that need to update statistics. - * TODO: we should reuse the Partitions generated at compile time - * since getting the list of partitions is quite expensive. - * - * @return a list of partitions that need to update statistics. - * @throws HiveException - */ - private List getPartitionsList(Hive db) throws HiveException { - if (work.getLoadFileDesc() != null) { - return null; //we are in CTAS, so we know there are no partitions - } - - List list = new ArrayList(); - - if (work.getTableSpecs() != null) { - - // ANALYZE command - TableSpec tblSpec = work.getTableSpecs(); - table = tblSpec.tableHandle; - if (!table.isPartitioned()) { - return null; - } - // get all partitions that matches with the partition spec - List partitions = tblSpec.partitions; - if (partitions != null) { - for (Partition partn : partitions) { - list.add(partn); - } - } - } else if (work.getLoadTableDesc() != null) { + private Collection dpPartSpecs; - // INSERT OVERWRITE command - LoadTableDesc tbd = work.getLoadTableDesc(); - table = db.getTable(tbd.getTable().getTableName()); - if (!table.isPartitioned()) { - return null; - } - DynamicPartitionCtx dpCtx = tbd.getDPCtx(); - if (dpCtx != null && dpCtx.getNumDPCols() > 0) { // dynamic partitions - // If no dynamic partitions are generated, dpPartSpecs may not be initialized - if (dpPartSpecs != null) { - // load the list of DP partitions and return the list of partition specs - list.addAll(dpPartSpecs); - } - } else { // static partition - Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); - list.add(partn); - } + @Override + protected void receiveFeed(FeedType feedType, Object feedValue) { + // this method should be called by MoveTask when there are dynamic + // partitions generated + if (feedType == FeedType.DYNAMIC_PARTITIONS) { + dpPartSpecs = (Collection) feedValue; } - return list; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index fe9b6244df..abfdd867b8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanTask; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.ConditionalWork; import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -54,8 +54,8 @@ import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.ReplCopyWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.TezWork; /** @@ -99,10 +99,9 @@ public TaskTuple(Class workClass, Class> taskClass) { taskvec.add(new TaskTuple(MapredLocalWork.class, MapredLocalTask.class)); - taskvec.add(new TaskTuple(StatsWork.class, - StatsTask.class)); - taskvec.add(new TaskTuple(StatsNoJobWork.class, StatsNoJobTask.class)); - taskvec.add(new TaskTuple(ColumnStatsWork.class, ColumnStatsTask.class)); + taskvec.add(new TaskTuple(BasicStatsWork.class, BasicStatsTask.class)); + taskvec.add(new TaskTuple(BasicStatsNoJobWork.class, BasicStatsNoJobTask.class)); + taskvec.add(new TaskTuple(StatsWork.class, StatsTask.class)); taskvec.add(new TaskTuple(ColumnStatsUpdateWork.class, ColumnStatsUpdateTask.class)); taskvec.add(new TaskTuple(MergeFileWork.class, MergeFileTask.class)); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 9f98b69b18..a7befd0b2e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1741,7 +1741,9 @@ public Partition loadPartition(Path loadPath, Table tbl, } // column stats will be inaccurate - StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); + if (!hasFollowingStatsTask) { + StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); + } // recreate the partition if it existed before if (isSkewedStoreAsSubdir) { @@ -1760,8 +1762,8 @@ public Partition loadPartition(Path loadPath, Table tbl, if (oldPart == null) { newTPart.getTPartition().setParameters(new HashMap()); if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), null, - StatsSetupConst.TRUE); + StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), + MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE); } MetaStoreUtils.populateQuickStats(HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf)), newTPart.getParameters()); try { @@ -2123,7 +2125,9 @@ public void loadTable(Path loadPath, String tableName, boolean replace, boolean } //column stats will be inaccurate - StatsSetupConst.clearColumnStatsState(tbl.getParameters()); + if (!hasFollowingStatsTask) { + StatsSetupConst.clearColumnStatsState(tbl.getParameters()); + } try { if (isSkewedStoreAsSubdir) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 61f6a7c4ff..e49c5ae3d5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -434,6 +434,8 @@ private void createTempTable(org.apache.hadoop.hive.metastore.api.Table tbl, // Add temp table info to current session Table tTable = new Table(tbl); + StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), + MetaStoreUtils.getColumnNamesForTable(tbl), StatsSetupConst.TRUE); if (tables == null) { tables = new HashMap(); ss.getTempTables().put(dbName, tables); @@ -711,6 +713,13 @@ private boolean updateTempTableColumnStats(String dbName, String tableName, ssTableColStats); } mergeColumnStats(ssTableColStats, colStats); + + List colNames = new ArrayList<>(); + for (ColumnStatisticsObj obj : colStats.getStatsObj()) { + colNames.add(obj.getColName()); + } + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tableName); + StatsSetupConst.setColumnStatsState(table.getParameters(), colNames); return true; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java index 9297a0b874..9f223af877 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; @@ -45,10 +46,11 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.mapred.InputFormat; /** @@ -99,7 +101,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any MR or Tez job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(op.getConf().getTableMetadata().getTableSpec()); + BasicStatsNoJobWork snjWork = new BasicStatsNoJobWork(op.getConf().getTableMetadata().getTableSpec()); snjWork.setStatsReliable(parseCtx.getConf().getBoolVar( HiveConf.ConfVars.HIVE_STATS_RELIABLE)); // If partition is specified, get pruned partition list @@ -111,7 +113,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, partCols, false); snjWork.setPrunedPartitionList(partList); } - Task snjTask = TaskFactory.get(snjWork, parseCtx.getConf()); + Task snjTask = TaskFactory.get(snjWork, parseCtx.getConf()); ctx.setCurrTask(snjTask); ctx.setCurrTopOp(null); ctx.getRootTasks().clear(); @@ -121,14 +123,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, // The plan consists of a simple MapRedTask followed by a StatsTask. // The MR task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(op.getConf().getTableMetadata().getTableSpec()); + BasicStatsWork statsWork = new BasicStatsWork(op.getConf().getTableMetadata().getTableSpec()); statsWork.setAggKey(op.getConf().getStatsAggPrefix()); statsWork.setStatsTmpDir(op.getConf().getTmpStatsDir()); statsWork.setSourceTask(currTask); statsWork.setStatsReliable(parseCtx.getConf().getBoolVar( HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseCtx.getConf()); - currTask.addDependentTask(statsTask); + StatsWork columnStatsWork = new StatsWork(statsWork); + Task columnStatsTask = TaskFactory.get(columnStatsWork, parseCtx.getConf()); + currTask.addDependentTask(columnStatsTask); if (!ctx.getRootTasks().contains(currTask)) { ctx.getRootTasks().add(currTask); } @@ -136,15 +139,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // The plan consists of a StatsTask only. if (noScan) { - statsTask.setParentTasks(null); + columnStatsTask.setParentTasks(null); statsWork.setNoScanAnalyzeCommand(true); ctx.getRootTasks().remove(currTask); - ctx.getRootTasks().add(statsTask); + ctx.getRootTasks().add(columnStatsTask); } // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan; if (partialScan) { - handlePartialScanCommand(op, ctx, parseCtx, currTask, statsWork, statsTask); + handlePartialScanCommand(op, ctx, parseCtx, currTask, statsWork, columnStatsTask); } currWork.getMapWork().setGatheringStats(true); @@ -188,7 +191,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, */ private void handlePartialScanCommand(TableScanOperator op, GenMRProcContext ctx, ParseContext parseCtx, Task currTask, - StatsWork statsWork, Task statsTask) throws SemanticException { + BasicStatsWork statsWork, Task statsTask) throws SemanticException { String aggregationKey = op.getConf().getStatsAggPrefix(); StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey); List inputPaths = GenMapRedUtils.getInputPathsForPartialScan(op, aggregationKeyBuffer); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index da153e36d2..0286f1343a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -35,6 +35,7 @@ import java.util.Set; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.BlobStorageUtils; @@ -88,6 +89,7 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles; import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles.ConditionalResolverMergeFilesCtx; import org.apache.hadoop.hive.ql.plan.ConditionalWork; @@ -111,7 +113,7 @@ import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.TezWork; @@ -498,6 +500,10 @@ public static void setMapWork(MapWork plan, ParseContext parseCtx, Set currTask, HiveConf hconf) { MoveWork mvWork = mvTask.getWork(); - StatsWork statsWork = null; + BasicStatsWork statsWork = null; if (mvWork.getLoadTableWork() != null) { - statsWork = new StatsWork(mvWork.getLoadTableWork()); + statsWork = new BasicStatsWork(mvWork.getLoadTableWork()); } else if (mvWork.getLoadFileWork() != null) { - statsWork = new StatsWork(mvWork.getLoadFileWork()); + statsWork = new BasicStatsWork(mvWork.getLoadFileWork()); } assert statsWork != null : "Error when generating StatsTask"; @@ -1506,7 +1512,8 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix // in FileSinkDesc is used for stats publishing. They should be consistent. statsWork.setAggKey(nd.getConf().getStatsAggPrefix()); - Task statsTask = TaskFactory.get(statsWork, hconf); + StatsWork columnStatsWork = new StatsWork(statsWork); + Task statsTask = TaskFactory.get(columnStatsWork, hconf); // subscribe feeds from the MoveTask so that MoveTask can forward the list // of dynamic partition list to the StatsTask diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java index 3a20cfe7ac..eae9505709 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java @@ -35,10 +35,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.BasicStatsTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.tez.DagUtils; import org.apache.hadoop.hive.ql.exec.tez.TezTask; @@ -92,8 +93,9 @@ public MemoryCalculator(PhysicalContext pctx) { public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) throws SemanticException { Task currTask = (Task) nd; - if (currTask instanceof StatsTask) { - currTask = ((StatsTask) currTask).getWork().getSourceTask(); + if (currTask instanceof StatsTask + && ((StatsTask) currTask).getWork().getBasicStatsWork() != null) { + currTask = ((StatsTask) currTask).getWork().getBasicStatsWork().getSourceTask(); } if (currTask instanceof TezTask) { TezWork work = ((TezTask) currTask).getWork(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java index dc433fed22..f3c6daedfc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java @@ -26,8 +26,9 @@ import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.SerializationUtilities; +import org.apache.hadoop.hive.ql.exec.BasicStatsTask; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.tez.TezTask; @@ -71,8 +72,9 @@ public Serializer(PhysicalContext pctx) { public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) throws SemanticException { Task currTask = (Task) nd; - if (currTask instanceof StatsTask) { - currTask = ((StatsTask) currTask).getWork().getSourceTask(); + if (currTask instanceof StatsTask + && ((StatsTask) currTask).getWork().getBasicStatsWork() != null) { + currTask = ((StatsTask) currTask).getWork().getBasicStatsWork().getSourceTask(); } if (currTask instanceof TezTask) { TezWork work = ((TezTask) currTask).getWork(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 251decac9b..9e2015221e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -91,6 +91,7 @@ import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc; import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -130,7 +131,7 @@ import org.apache.hadoop.hive.ql.plan.ShowTablesDesc; import org.apache.hadoop.hive.ql.plan.ShowTblPropertiesDesc; import org.apache.hadoop.hive.ql.plan.ShowTxnsDesc; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TruncateTableDesc; @@ -1091,18 +1092,19 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { // Recalculate the HDFS stats if auto gather stats is set if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc; + BasicStatsWork basicStatsWork; if (oldTblPartLoc.equals(newTblPartLoc)) { // If we're merging to the same location, we can avoid some metastore calls TableSpec tablepart = new TableSpec(this.db, conf, root); - statDesc = new StatsWork(tablepart); + basicStatsWork = new BasicStatsWork(tablepart); } else { - statDesc = new StatsWork(ltd); + basicStatsWork = new BasicStatsWork(ltd); } - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statTask = TaskFactory.get(statDesc, conf); + basicStatsWork.setNoStatsAggregator(true); + basicStatsWork.setClearAggregatorStats(true); + basicStatsWork.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork columnStatsWork = new StatsWork(basicStatsWork); + Task statTask = TaskFactory.get(columnStatsWork, conf); moveTsk.addDependentTask(statTask); } } catch (HiveException e) { @@ -1724,18 +1726,19 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, mergeTask.addDependentTask(moveTsk); if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc; + BasicStatsWork basicStatsWork; if (oldTblPartLoc.equals(newTblPartLoc)) { // If we're merging to the same location, we can avoid some metastore calls TableSpec tableSpec = new TableSpec(db, tableName, partSpec); - statDesc = new StatsWork(tableSpec); + basicStatsWork = new BasicStatsWork(tableSpec); } else { - statDesc = new StatsWork(ltd); + basicStatsWork = new BasicStatsWork(ltd); } - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statTask = TaskFactory.get(statDesc, conf); + basicStatsWork.setNoStatsAggregator(true); + basicStatsWork.setClearAggregatorStats(true); + basicStatsWork.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork columnStatsWork = new StatsWork(basicStatsWork); + Task statTask = TaskFactory.get(columnStatsWork, conf); moveTsk.addDependentTask(statTask); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index 7a0d4a752e..fbef0dd837 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.BasicStatsTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 40d4fad314..39c027a8cf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -205,10 +205,6 @@ protected void setupMapWork(MapWork mapWork, GenTezProcContext context, // All the setup is done in GenMapRedUtils GenMapRedUtils.setMapWork(mapWork, context.parseContext, context.inputs, partitions, root, alias, context.conf, false); - // we also collect table stats while collecting column stats. - if (context.parseContext.getAnalyzeRewrite() != null) { - mapWork.setGatheringStats(true); - } } // removes any union operator and clones the plan diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index fa79700df7..6a7b3db17a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; - import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import java.io.IOException; @@ -51,9 +50,10 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.MoveWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.mapred.InputFormat; import com.google.common.collect.Lists; @@ -301,11 +301,12 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // Update the stats which do not require a complete scan. Task statTask = null; if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc = new StatsWork(loadTableWork); - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - statTask = TaskFactory.get(statDesc, conf); + BasicStatsWork basicStatsWork = new BasicStatsWork(loadTableWork); + basicStatsWork.setNoStatsAggregator(true); + basicStatsWork.setClearAggregatorStats(true); + basicStatsWork.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork columnStatsWork = new StatsWork(basicStatsWork); + statTask = TaskFactory.get(columnStatsWork, conf); } // HIVE-3334 has been filed for load file with index auto update diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java index b6d7ee8a92..abc7ed29a4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java @@ -44,9 +44,10 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; -import org.apache.hadoop.hive.ql.plan.MapWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.MapWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.mapred.InputFormat; @@ -103,7 +104,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any Tez job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata() + BasicStatsNoJobWork snjWork = new BasicStatsNoJobWork(tableScan.getConf().getTableMetadata() .getTableSpec()); snjWork.setStatsReliable(parseContext.getConf().getBoolVar( HiveConf.ConfVars.HIVE_STATS_RELIABLE)); @@ -116,7 +117,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, false); snjWork.setPrunedPartitionList(partList); } - Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); + Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); snjTask.setParentTasks(null); context.rootTasks.remove(context.currentTask); context.rootTasks.add(snjTask); @@ -127,27 +128,28 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, // The plan consists of a simple TezTask followed by a StatsTask. // The Tez task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar( + BasicStatsWork basicStatsWork = new BasicStatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + basicStatsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); + basicStatsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); + basicStatsWork.setSourceTask(context.currentTask); + basicStatsWork.setStatsReliable(parseContext.getConf().getBoolVar( HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseContext.getConf()); + StatsWork columnStatsWork = new StatsWork(basicStatsWork); + Task statsTask = TaskFactory.get(columnStatsWork, parseContext.getConf()); context.currentTask.addDependentTask(statsTask); // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // The plan consists of a StatsTask only. if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) { statsTask.setParentTasks(null); - statsWork.setNoScanAnalyzeCommand(true); + columnStatsWork.getBasicStatsWork().setNoScanAnalyzeCommand(true); context.rootTasks.remove(context.currentTask); context.rootTasks.add(statsTask); } // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan; if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) { - handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask); + handlePartialScanCommand(tableScan, parseContext, basicStatsWork, context, statsTask); } // NOTE: here we should use the new partition predicate pushdown API to @@ -166,65 +168,18 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, return true; } - } else if (parseContext.getAnalyzeRewrite() != null) { - // we need to collect table stats while collecting column stats. - try { - context.currentTask.addDependentTask(genTableStats(context, tableScan)); - } catch (HiveException e) { - throw new SemanticException(e); - } } return null; } - private Task genTableStats(GenTezProcContext context, TableScanOperator tableScan) - throws HiveException { - Class inputFormat = tableScan.getConf().getTableMetadata() - .getInputFormatClass(); - ParseContext parseContext = context.parseContext; - Table table = tableScan.getConf().getTableMetadata(); - List partitions = new ArrayList<>(); - if (table.isPartitioned()) { - partitions.addAll(parseContext.getPrunedPartitions(tableScan).getPartitions()); - for (Partition partn : partitions) { - LOG.debug("XXX: adding part: " + partn); - context.outputs.add(new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK)); - } - } - TableSpec tableSpec = new TableSpec(table, partitions); - tableScan.getConf().getTableMetadata().setTableSpec(tableSpec); - - if (inputFormat.equals(OrcInputFormat.class)) { - // For ORC, there is no Tez Job for table stats. - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata() - .getTableSpec()); - snjWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - // If partition is specified, get pruned partition list - if (partitions.size() > 0) { - snjWork.setPrunedPartitionList(parseContext.getPrunedPartitions(tableScan)); - } - return TaskFactory.get(snjWork, parseContext.getConf()); - } else { - - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - return TaskFactory.get(statsWork, parseContext.getConf()); - } - } - /** * handle partial scan command. * * It is composed of PartialScanTask followed by StatsTask. */ private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext, - StatsWork statsWork, GenTezProcContext context, Task statsTask) + BasicStatsWork statsWork, GenTezProcContext context, Task statsTask) throws SemanticException { String aggregationKey = tableScan.getConf().getStatsAggPrefix(); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 1c74779dec..de07ef24d5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7330,7 +7330,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // the following code is used to collect column stats when // hive.stats.autogather=true // and it is an insert overwrite or insert into table - if (dest_tab != null && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) + if (dest_tab != null && !dest_tab.isNonNative() + && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER) && ColumnStatsAutoGatherContext.canRunAutogatherStats(fso)) { if (dest_type.intValue() == QBMetaData.DEST_TABLE) { @@ -10474,10 +10475,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String throws SemanticException { // if it is not analyze command and not column stats, then do not gatherstats - // if it is column stats, but it is not tez, do not gatherstats - if ((!qbp.isAnalyzeCommand() && qbp.getAnalyzeRewrite() == null) - || (qbp.getAnalyzeRewrite() != null && !HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))) { + if (!qbp.isAnalyzeCommand() && qbp.getAnalyzeRewrite() == null) { tsDesc.setGatherStats(false); } else { if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 08a8f00e06..9ef2fe367d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -21,11 +21,14 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.Stack; @@ -35,15 +38,17 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.exec.ColumnStatsTask; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.BasicStatsTask; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -51,13 +56,19 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -76,6 +87,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.thrift.ThriftFormatter; import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe; +import org.apache.hadoop.mapred.InputFormat; import com.google.common.collect.Interner; import com.google.common.collect.Interners; @@ -294,18 +306,53 @@ public void compile(final ParseContext pCtx, final List> leafTasks = new LinkedHashSet>(); - getLeafTasks(rootTasks, leafTasks); + // map from tablename to task (ColumnStatsTask which includes a BasicStatsTask) + Map map = new LinkedHashMap<>(); if (isCStats) { - genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadFileWork, leafTasks, outerQueryLimit, 0); + if (rootTasks == null || rootTasks.size() != 1 || pCtx.getTopOps() == null + || pCtx.getTopOps().size() != 1) { + throw new SemanticException("Can not find correct root task!"); + } + try { + Task root = rootTasks.iterator().next(); + StatsTask tsk = (StatsTask) genTableStats(pCtx, pCtx.getTopOps().values() + .iterator().next(), root, outputs); + root.addDependentTask(tsk); + map.put(extractTableFullName((StatsTask) tsk), (StatsTask) tsk); + } catch (HiveException e) { + throw new SemanticException(e); + } + genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadFileWork, map, outerQueryLimit, 0); } else { + Set> leafTasks = new LinkedHashSet>(); + getLeafTasks(rootTasks, leafTasks); + List> nonStatsLeafTasks = new ArrayList<>(); + for (Task tsk : leafTasks) { + // map table name to the correct ColumnStatsTask + if (tsk instanceof StatsTask) { + map.put(extractTableFullName((StatsTask) tsk), (StatsTask) tsk); + } else { + nonStatsLeafTasks.add(tsk); + } + } + // add cStatsTask as a dependent of all the nonStatsLeafTasks + for (Task tsk : nonStatsLeafTasks) { + for (Task cStatsTask : map.values()) { + tsk.addDependentTask(cStatsTask); + } + } for (ColumnStatsAutoGatherContext columnStatsAutoGatherContext : pCtx .getColumnStatsAutoGatherContexts()) { if (!columnStatsAutoGatherContext.isInsertInto()) { genColumnStatsTask(columnStatsAutoGatherContext.getAnalyzeRewrite(), - columnStatsAutoGatherContext.getLoadFileWork(), leafTasks, outerQueryLimit, 0); + columnStatsAutoGatherContext.getLoadFileWork(), map, outerQueryLimit, 0); } else { int numBitVector; try { @@ -314,7 +361,7 @@ public void compile(final ParseContext pCtx, final List genTableStats(ParseContext parseContext, TableScanOperator tableScan, Task currentTask, final HashSet outputs) + throws HiveException { + Class inputFormat = tableScan.getConf().getTableMetadata() + .getInputFormatClass(); + Table table = tableScan.getConf().getTableMetadata(); + List partitions = new ArrayList<>(); + if (table.isPartitioned()) { + partitions.addAll(parseContext.getPrunedPartitions(tableScan).getPartitions()); + for (Partition partn : partitions) { + LOG.debug("XXX: adding part: " + partn); + outputs.add(new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK)); + } + } + TableSpec tableSpec = new TableSpec(table, partitions); + tableScan.getConf().getTableMetadata().setTableSpec(tableSpec); + + if (inputFormat.equals(OrcInputFormat.class)) { + // For ORC, there is no Tez Job for table stats. + BasicStatsNoJobWork snjWork = new BasicStatsNoJobWork(tableScan.getConf().getTableMetadata() + .getTableSpec()); + snjWork.setStatsReliable(parseContext.getConf().getBoolVar( + HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + // If partition is specified, get pruned partition list + if (partitions.size() > 0) { + snjWork.setPrunedPartitionList(parseContext.getPrunedPartitions(tableScan)); + } + StatsWork columnStatsWork = new StatsWork(snjWork); + return TaskFactory.get(columnStatsWork, parseContext.getConf()); + } else { + BasicStatsWork statsWork = new BasicStatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); + statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); + statsWork.setSourceTask(currentTask); + statsWork.setStatsReliable(parseContext.getConf().getBoolVar( + HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork columnStatsWork = new StatsWork(statsWork); + return TaskFactory.get(columnStatsWork, parseContext.getConf()); + } + } + private void patchUpAfterCTASorMaterializedView(final List> rootTasks, final HashSet outputs, Task createTask) { @@ -388,7 +496,8 @@ private void patchUpAfterCTASorMaterializedView(final List 0); for (Task task : leaves) { - if (task instanceof StatsTask) { + if (task instanceof StatsTask + && ((StatsTask) task).getWork().getBasicStatsWork() != null) { // StatsTask require table to already exist for (Task parentOfStatsTask : task.getParentTasks()) { parentOfStatsTask.addDependentTask(createTask); @@ -416,13 +525,12 @@ private void patchUpAfterCTASorMaterializedView(final List loadFileWork, Set> leafTasks, - int outerQueryLimit, int numBitVector) { - ColumnStatsTask cStatsTask = null; - ColumnStatsWork cStatsWork = null; + List loadFileWork, Map map, + int outerQueryLimit, int numBitVector) throws SemanticException { FetchWork fetch = null; String tableName = analyzeRewrite.getTableName(); List colName = analyzeRewrite.getColName(); @@ -450,10 +558,12 @@ protected void genColumnStatsTask(AnalyzeRewriteContext analyzeRewrite, ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName, colName, colType, isTblLevel, numBitVector); - cStatsWork = new ColumnStatsWork(fetch, cStatsDesc); - cStatsTask = (ColumnStatsTask) TaskFactory.get(cStatsWork, conf); - for (Task tsk : leafTasks) { - tsk.addDependentTask(cStatsTask); + StatsTask columnStatsTask = map.get(tableName); + if (columnStatsTask == null) { + throw new SemanticException("Can not find " + tableName + " in genColumnStatsTask"); + } else { + columnStatsTask.getWork().setfWork(fetch); + columnStatsTask.getWork().setColStats(cStatsDesc); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java index a2876e1d4f..74162c20f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java @@ -42,10 +42,11 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.mapred.InputFormat; import com.google.common.base.Preconditions; @@ -106,7 +107,7 @@ public Object process(Node nd, Stack stack, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan; // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any Spark job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec()); + BasicStatsNoJobWork snjWork = new BasicStatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec()); snjWork.setStatsReliable(parseContext.getConf().getBoolVar( HiveConf.ConfVars.HIVE_STATS_RELIABLE)); // If partition is specified, get pruned partition list @@ -118,7 +119,7 @@ public Object process(Node nd, Stack stack, false); snjWork.setPrunedPartitionList(partList); } - Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); + Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); snjTask.setParentTasks(null); context.rootTasks.remove(context.currentTask); context.rootTasks.add(snjTask); @@ -129,26 +130,27 @@ public Object process(Node nd, Stack stack, // The plan consists of a simple SparkTask followed by a StatsTask. // The Spark task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseContext.getConf()); + BasicStatsWork basicStatsWork = new BasicStatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + basicStatsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); + basicStatsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); + basicStatsWork.setSourceTask(context.currentTask); + basicStatsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork columnStatsWork = new StatsWork(basicStatsWork); + Task statsTask = TaskFactory.get(columnStatsWork, parseContext.getConf()); context.currentTask.addDependentTask(statsTask); // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // The plan consists of a StatsTask only. if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) { statsTask.setParentTasks(null); - statsWork.setNoScanAnalyzeCommand(true); + basicStatsWork.setNoScanAnalyzeCommand(true); context.rootTasks.remove(context.currentTask); context.rootTasks.add(statsTask); } // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan; if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) { - handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask); + handlePartialScanCommand(tableScan, parseContext, basicStatsWork, context, statsTask); } // NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list, @@ -176,7 +178,7 @@ public Object process(Node nd, Stack stack, * It is composed of PartialScanTask followed by StatsTask. */ private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext, - StatsWork statsWork, GenSparkProcContext context, Task statsTask) + BasicStatsWork statsWork, GenSparkProcContext context, Task statsTask) throws SemanticException { String aggregationKey = tableScan.getConf().getStatsAggPrefix(); StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsNoJobWork.java similarity index 83% rename from ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java rename to ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsNoJobWork.java index 77c04f6c6e..db33ac400d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsNoJobWork.java @@ -22,29 +22,27 @@ import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; -import org.apache.hadoop.hive.ql.plan.Explain.Level; /** * Client-side stats aggregator task. */ -@Explain(displayName = "Stats-Aggr Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class StatsNoJobWork implements Serializable { +public class BasicStatsNoJobWork implements Serializable { private static final long serialVersionUID = 1L; private TableSpec tableSpecs; private boolean statsReliable; private PrunedPartitionList prunedPartitionList; - public StatsNoJobWork() { + public BasicStatsNoJobWork() { } - public StatsNoJobWork(TableSpec tableSpecs) { + public BasicStatsNoJobWork(TableSpec tableSpecs) { this.tableSpecs = tableSpecs; } - public StatsNoJobWork(boolean statsReliable) { + public BasicStatsNoJobWork(boolean statsReliable) { this.statsReliable = statsReliable; } @@ -67,4 +65,5 @@ public void setPrunedPartitionList(PrunedPartitionList prunedPartitionList) { public PrunedPartitionList getPrunedPartitionList() { return prunedPartitionList; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java new file mode 100644 index 0000000000..24dbff119a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + + +/** + * ConditionalStats. + * + */ +public class BasicStatsWork implements Serializable { + private static final long serialVersionUID = 1L; + + private TableSpec tableSpecs; // source table spec -- for TableScanOperator + private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator + private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator + private String aggKey; // aggregation key prefix + private boolean statsReliable; // are stats completely reliable + + // If stats aggregator is not present, clear the current aggregator stats. + // For eg. if a merge is being performed, stats already collected by aggregator (numrows etc.) + // are still valid. However, if a load file is being performed, the old stats collected by + // aggregator are not valid. It might be a good idea to clear them instead of leaving wrong + // and old stats. + // Since HIVE-12661, we maintain the old stats (although may be wrong) for CBO + // purpose. We use a flag COLUMN_STATS_ACCURATE to + // show the accuracy of the stats. + + private boolean clearAggregatorStats = false; + + private boolean noStatsAggregator = false; + + private boolean isNoScanAnalyzeCommand = false; + + private boolean isPartialScanAnalyzeCommand = false; + + // sourceTask for TS is not changed (currently) but that of FS might be changed + // by various optimizers (auto.convert.join, for example) + // so this is set by DriverContext in runtime + private transient Task sourceTask; + + private boolean isFollowedByColStats = false; + + // used by FS based stats collector + private String statsTmpDir; + + public BasicStatsWork() { + } + + public BasicStatsWork(TableSpec tableSpecs) { + this.tableSpecs = tableSpecs; + } + + public BasicStatsWork(LoadTableDesc loadTableDesc) { + this.loadTableDesc = loadTableDesc; + } + + public BasicStatsWork(LoadFileDesc loadFileDesc) { + this.loadFileDesc = loadFileDesc; + } + + public TableSpec getTableSpecs() { + return tableSpecs; + } + + public LoadTableDesc getLoadTableDesc() { + return loadTableDesc; + } + + public LoadFileDesc getLoadFileDesc() { + return loadFileDesc; + } + + public void setAggKey(String aggK) { + aggKey = aggK; + } + + @Explain(displayName = "Stats Aggregation Key Prefix", explainLevels = { Level.EXTENDED }) + public String getAggKey() { + return aggKey; + } + + public String getStatsTmpDir() { + return statsTmpDir; + } + + public void setStatsTmpDir(String statsTmpDir) { + this.statsTmpDir = statsTmpDir; + } + + public boolean getNoStatsAggregator() { + return noStatsAggregator; + } + + public void setNoStatsAggregator(boolean noStatsAggregator) { + this.noStatsAggregator = noStatsAggregator; + } + + public boolean isStatsReliable() { + return statsReliable; + } + + public void setStatsReliable(boolean statsReliable) { + this.statsReliable = statsReliable; + } + + public boolean isClearAggregatorStats() { + return clearAggregatorStats; + } + + public void setClearAggregatorStats(boolean clearAggregatorStats) { + this.clearAggregatorStats = clearAggregatorStats; + } + + /** + * @return the isNoScanAnalyzeCommand + */ + public boolean isNoScanAnalyzeCommand() { + return isNoScanAnalyzeCommand; + } + + /** + * @param isNoScanAnalyzeCommand the isNoScanAnalyzeCommand to set + */ + public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) { + this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand; + } + + /** + * @return the isPartialScanAnalyzeCommand + */ + public boolean isPartialScanAnalyzeCommand() { + return isPartialScanAnalyzeCommand; + } + + /** + * @param isPartialScanAnalyzeCommand the isPartialScanAnalyzeCommand to set + */ + public void setPartialScanAnalyzeCommand(boolean isPartialScanAnalyzeCommand) { + this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand; + } + + public Task getSourceTask() { + return sourceTask; + } + + public void setSourceTask(Task sourceTask) { + this.sourceTask = sourceTask; + } + + public boolean isFollowedByColStats() { + return isFollowedByColStats; + } + + public void setFollowedByColStats(boolean isFollowedByColStats) { + this.isFollowedByColStats = isFollowedByColStats; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java index 97f323f4b7..a756a29d8b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java @@ -30,6 +30,7 @@ private static final long serialVersionUID = 1L; private boolean isTblLevel; private int numBitVector; + private boolean needMerge; private String tableName; private List colName; private List colType; @@ -44,6 +45,7 @@ public ColumnStatsDesc(String tableName, List colName, List colT this.colType = colType; this.isTblLevel = isTblLevel; this.numBitVector = 0; + this.needMerge = false; } public ColumnStatsDesc(String tableName, List colName, @@ -53,6 +55,7 @@ public ColumnStatsDesc(String tableName, List colName, this.colType = colType; this.isTblLevel = isTblLevel; this.numBitVector = numBitVector; + this.needMerge = this.numBitVector != 0; } @Explain(displayName = "Table") @@ -99,4 +102,8 @@ public void setNumBitVector(int numBitVector) { this.numBitVector = numBitVector; } + public boolean isNeedMerge() { + return needMerge; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java deleted file mode 100644 index 76811b1a93..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; - -import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.ListSinkOperator; -import org.apache.hadoop.hive.ql.plan.Explain.Level; - - -/** - * ColumnStats Work. - * - */ -@Explain(displayName = "Column Stats Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class ColumnStatsWork implements Serializable { - private static final long serialVersionUID = 1L; - private FetchWork fWork; - private ColumnStatsDesc colStats; - private static final int LIMIT = -1; - - - public ColumnStatsWork() { - } - - public ColumnStatsWork(FetchWork work, ColumnStatsDesc colStats) { - this.fWork = work; - this.setColStats(colStats); - } - - @Override - public String toString() { - String ret; - ret = fWork.toString(); - return ret; - } - - public FetchWork getfWork() { - return fWork; - } - - public void setfWork(FetchWork fWork) { - this.fWork = fWork; - } - - @Explain(displayName = "Column Stats Desc") - public ColumnStatsDesc getColStats() { - return colStats; - } - - public void setColStats(ColumnStatsDesc colStats) { - this.colStats = colStats; - } - - public ListSinkOperator getSink() { - return fWork.getSink(); - } - - public void initializeForFetch(CompilationOpContext ctx) { - fWork.initializeForFetch(ctx); - } - - public int getLeastNumRows() { - return fWork.getLeastNumRows(); - } - - public static int getLimit() { - return LIMIT; - } - -} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java index a5050c5368..e6af6c7005 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java @@ -20,151 +20,97 @@ import java.io.Serializable; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.exec.ListSinkOperator; import org.apache.hadoop.hive.ql.plan.Explain.Level; - /** - * ConditionalStats. + * Stats Work, may include basic stats work and column stats desc * */ -@Explain(displayName = "Stats-Aggr Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +@Explain(displayName = "Stats Work", explainLevels = { Level.USER, Level.DEFAULT, + Level.EXTENDED }) public class StatsWork implements Serializable { private static final long serialVersionUID = 1L; - - private TableSpec tableSpecs; // source table spec -- for TableScanOperator - private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator - private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator - private String aggKey; // aggregation key prefix - private boolean statsReliable; // are stats completely reliable - - // If stats aggregator is not present, clear the current aggregator stats. - // For eg. if a merge is being performed, stats already collected by aggregator (numrows etc.) - // are still valid. However, if a load file is being performed, the old stats collected by - // aggregator are not valid. It might be a good idea to clear them instead of leaving wrong - // and old stats. - // Since HIVE-12661, we maintain the old stats (although may be wrong) for CBO - // purpose. We use a flag COLUMN_STATS_ACCURATE to - // show the accuracy of the stats. - - private boolean clearAggregatorStats = false; - - private boolean noStatsAggregator = false; - - private boolean isNoScanAnalyzeCommand = false; - - private boolean isPartialScanAnalyzeCommand = false; - - // sourceTask for TS is not changed (currently) but that of FS might be changed - // by various optimizers (auto.convert.join, for example) - // so this is set by DriverContext in runtime - private transient Task sourceTask; - - // used by FS based stats collector - private String statsTmpDir; + // this is for basic stats + private BasicStatsWork basicStatsWork; + private BasicStatsNoJobWork basicStatsNoJobWork; + private FetchWork fWork; + private ColumnStatsDesc colStats; + private static final int LIMIT = -1; public StatsWork() { } - public StatsWork(TableSpec tableSpecs) { - this.tableSpecs = tableSpecs; - } - - public StatsWork(LoadTableDesc loadTableDesc) { - this.loadTableDesc = loadTableDesc; - } - - public StatsWork(LoadFileDesc loadFileDesc) { - this.loadFileDesc = loadFileDesc; - } - - public TableSpec getTableSpecs() { - return tableSpecs; + public StatsWork(BasicStatsWork basicStatsWork) { + super(); + this.basicStatsWork = basicStatsWork; } - public LoadTableDesc getLoadTableDesc() { - return loadTableDesc; + public StatsWork(BasicStatsNoJobWork basicStatsNoJobWork) { + super(); + this.basicStatsNoJobWork = basicStatsNoJobWork; } - public LoadFileDesc getLoadFileDesc() { - return loadFileDesc; + public StatsWork(FetchWork work, ColumnStatsDesc colStats) { + this.fWork = work; + this.setColStats(colStats); } - public void setAggKey(String aggK) { - aggKey = aggK; + @Override + public String toString() { + return String.format("StatWork; fetch: %s", fWork ); } - @Explain(displayName = "Stats Aggregation Key Prefix", explainLevels = { Level.EXTENDED }) - public String getAggKey() { - return aggKey; + public FetchWork getfWork() { + return fWork; } - public String getStatsTmpDir() { - return statsTmpDir; + public void setfWork(FetchWork fWork) { + this.fWork = fWork; } - public void setStatsTmpDir(String statsTmpDir) { - this.statsTmpDir = statsTmpDir; + @Explain(displayName = "Column Stats Desc") + public ColumnStatsDesc getColStats() { + return colStats; } - public boolean getNoStatsAggregator() { - return noStatsAggregator; + public void setColStats(ColumnStatsDesc colStats) { + this.colStats = colStats; } - public void setNoStatsAggregator(boolean noStatsAggregator) { - this.noStatsAggregator = noStatsAggregator; + public ListSinkOperator getSink() { + return fWork.getSink(); } - public boolean isStatsReliable() { - return statsReliable; + public void initializeForFetch(CompilationOpContext ctx) { + fWork.initializeForFetch(ctx); } - public void setStatsReliable(boolean statsReliable) { - this.statsReliable = statsReliable; + public int getLeastNumRows() { + return fWork.getLeastNumRows(); } - public boolean isClearAggregatorStats() { - return clearAggregatorStats; + public static int getLimit() { + return LIMIT; } - public void setClearAggregatorStats(boolean clearAggregatorStats) { - this.clearAggregatorStats = clearAggregatorStats; + @Explain(displayName = "Basic Stats Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public BasicStatsWork getBasicStatsWork() { + return basicStatsWork; } - /** - * @return the isNoScanAnalyzeCommand - */ - public boolean isNoScanAnalyzeCommand() { - return isNoScanAnalyzeCommand; + public void setBasicStatsWork(BasicStatsWork basicStatsWork) { + this.basicStatsWork = basicStatsWork; } - /** - * @param isNoScanAnalyzeCommand the isNoScanAnalyzeCommand to set - */ - public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) { - this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand; + @Explain(displayName = "Basic Stats NoJob Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public BasicStatsNoJobWork getBasicStatsNoJobWork() { + return basicStatsNoJobWork; } - /** - * @return the isPartialScanAnalyzeCommand - */ - public boolean isPartialScanAnalyzeCommand() { - return isPartialScanAnalyzeCommand; + public void setBasicStatsNoJobWork(BasicStatsNoJobWork basicStatsNoJobWork) { + this.basicStatsNoJobWork = basicStatsNoJobWork; } - /** - * @param isPartialScanAnalyzeCommand the isPartialScanAnalyzeCommand to set - */ - public void setPartialScanAnalyzeCommand(boolean isPartialScanAnalyzeCommand) { - this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand; - } - - public Task getSourceTask() { - return sourceTask; - } - - public void setSourceTask(Task sourceTask) { - this.sourceTask = sourceTask; - } } diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 4f1c7d8b1e..44ce74d8bb 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -853,4 +853,4 @@ public void testMoreBucketsThanReducers2() throws Exception { int[][] expected = {{0, -1},{0, -1}, {1, -1}, {1, -1}, {2, -1}, {2, -1}, {3, -1}, {3, -1}}; Assert.assertEquals(stringifyValues(expected), r); } -} \ No newline at end of file +} diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 21b4a2ce55..ad8d39244b 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -129,6 +129,7 @@ protected void setUpWithTableProperties(String tableProperties) throws Exception .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); TxnDbUtil.setConfValues(hiveConf); TxnDbUtil.prepDb(); diff --git ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index ad2aac5f56..9062d924c4 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -64,6 +64,7 @@ void setUpInternal() throws Exception { .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); TxnDbUtil.setConfValues(hiveConf); TxnDbUtil.prepDb(); File f = new File(getWarehouseDir()); diff --git ql/src/test/queries/clientpositive/autoColumnStats_1.q ql/src/test/queries/clientpositive/autoColumnStats_1.q index 7955b07233..cc32393e65 100644 --- ql/src/test/queries/clientpositive/autoColumnStats_1.q +++ ql/src/test/queries/clientpositive/autoColumnStats_1.q @@ -60,6 +60,8 @@ drop table nzhang_part14; create table if not exists nzhang_part14 (key string) partitioned by (value string); +desc formatted nzhang_part14; + insert overwrite table nzhang_part14 partition(value) select key, value from ( select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a @@ -69,6 +71,8 @@ select key, value from ( select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T; +desc formatted nzhang_part14 partition (value=' '); + explain select key from nzhang_part14; diff --git ql/src/test/queries/clientpositive/autoColumnStats_10.q ql/src/test/queries/clientpositive/autoColumnStats_10.q new file mode 100644 index 0000000000..bf166d8701 --- /dev/null +++ ql/src/test/queries/clientpositive/autoColumnStats_10.q @@ -0,0 +1,52 @@ +set hive.mapred.mode=nonstrict; +set hive.stats.column.autogather=true; + +drop table p; + +CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint); + +desc formatted p; + +insert into p values (1,22,333); + +desc formatted p; + +alter table p replace columns (insert_num int, c1 STRING, c2 STRING); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; + +insert into p values (2,11,111); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; + +set hive.stats.column.autogather=false; + +drop table p; + +CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint); + +desc formatted p; + +insert into p values (1,22,333); + +desc formatted p; + +alter table p replace columns (insert_num int, c1 STRING, c2 STRING); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; + +insert into p values (2,11,111); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; diff --git ql/src/test/queries/clientpositive/autoColumnStats_5a.q ql/src/test/queries/clientpositive/autoColumnStats_5a.q new file mode 100644 index 0000000000..a8bce18e92 --- /dev/null +++ ql/src/test/queries/clientpositive/autoColumnStats_5a.q @@ -0,0 +1,30 @@ +set hive.stats.column.autogather=true; +set hive.mapred.mode=nonstrict; +set hive.cli.print.header=true; +SET hive.exec.schema.evolution=true; +SET hive.vectorized.execution.enabled=false; +set hive.fetch.task.conversion=none; +set hive.exec.dynamic.partition.mode=nonstrict; + +-- SORT_QUERY_RESULTS + +CREATE TABLE partitioned1(a INT, b STRING) PARTITIONED BY(part INT) STORED AS TEXTFILE; + +explain extended +insert into table partitioned1 partition(part=1) values(1, 'original'); + +insert into table partitioned1 partition(part=1) values(1, 'original'); + +desc formatted partitioned1 partition(part=1); + +explain extended +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original'); + +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original'); + +explain insert into table partitioned1 partition(part=1) values(1, 'original'),(2, 'original'), (3, 'original'),(4, 'original'); + +desc formatted partitioned1; +desc formatted partitioned1 partition(part=1); +desc formatted partitioned1 partition(part=1) a; + diff --git ql/src/test/queries/clientpositive/bucket_map_join_tez2.q ql/src/test/queries/clientpositive/bucket_map_join_tez2.q index 37989ecc9d..c600f639da 100644 --- ql/src/test/queries/clientpositive/bucket_map_join_tez2.q +++ ql/src/test/queries/clientpositive/bucket_map_join_tez2.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.strict.checks.bucketing=false; set hive.mapred.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/bucket_num_reducers.q ql/src/test/queries/clientpositive/bucket_num_reducers.q index 06f334e833..5c5008eea7 100644 --- ql/src/test/queries/clientpositive/bucket_num_reducers.q +++ ql/src/test/queries/clientpositive/bucket_num_reducers.q @@ -1,4 +1,4 @@ -; +set hive.stats.column.autogather=false; set hive.exec.mode.local.auto=false; set mapred.reduce.tasks = 10; diff --git ql/src/test/queries/clientpositive/combine1.q ql/src/test/queries/clientpositive/combine1.q index 3bcb8b19c1..b300830884 100644 --- ql/src/test/queries/clientpositive/combine1.q +++ ql/src/test/queries/clientpositive/combine1.q @@ -7,6 +7,8 @@ set mapred.max.split.size=256; set mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec; +set hive.stats.column.autogather=false; + -- SORT_QUERY_RESULTS create table combine1_1(key string, value string) stored as textfile; diff --git ql/src/test/queries/clientpositive/correlationoptimizer5.q ql/src/test/queries/clientpositive/correlationoptimizer5.q index 45b8cb955d..002fb12e22 100644 --- ql/src/test/queries/clientpositive/correlationoptimizer5.q +++ ql/src/test/queries/clientpositive/correlationoptimizer5.q @@ -1,3 +1,5 @@ +set hive.stats.column.autogather=false; +-- Currently, a query with multiple FileSinkOperators are not supported. set hive.mapred.mode=nonstrict; CREATE TABLE T1(key INT, val STRING); LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE T1; diff --git ql/src/test/queries/clientpositive/encryption_insert_values.q ql/src/test/queries/clientpositive/encryption_insert_values.q index 2dd3e9ad1d..c8d1d519f3 100644 --- ql/src/test/queries/clientpositive/encryption_insert_values.q +++ ql/src/test/queries/clientpositive/encryption_insert_values.q @@ -1,4 +1,5 @@ -- SORT_QUERY_RESULTS; +set hive.stats.column.autogather=false; DROP TABLE IF EXISTS encrypted_table PURGE; CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; @@ -12,4 +13,4 @@ select * from encrypted_table; -- this checks that we've actually created temp table data under encrypted_table folder describe formatted values__tmp__table__1; -CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file +CRYPTO DELETE_KEY --keyName key_128; diff --git ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q index 4dcea1f7ce..7159ad5995 100644 --- ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q +++ ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; --SORT_QUERY_RESULTS -- Java JCE must be installed in order to hava a key length of 256 bits diff --git ql/src/test/queries/clientpositive/encryption_move_tbl.q ql/src/test/queries/clientpositive/encryption_move_tbl.q index 0b7771cc4a..8d865aa6e8 100644 --- ql/src/test/queries/clientpositive/encryption_move_tbl.q +++ ql/src/test/queries/clientpositive/encryption_move_tbl.q @@ -1,4 +1,5 @@ -- SORT_QUERY_RESULTS; +set hive.stats.column.autogather=false; -- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after ALTER TABLE fails; diff --git ql/src/test/queries/clientpositive/exec_parallel_column_stats.q ql/src/test/queries/clientpositive/exec_parallel_column_stats.q index ceacc2442d..a89b707918 100644 --- ql/src/test/queries/clientpositive/exec_parallel_column_stats.q +++ ql/src/test/queries/clientpositive/exec_parallel_column_stats.q @@ -1,5 +1,7 @@ set hive.exec.parallel=true; -explain analyze table src compute statistics for columns; +create table t as select * from src; -analyze table src compute statistics for columns; \ No newline at end of file +explain analyze table t compute statistics for columns; + +analyze table t compute statistics for columns; diff --git ql/src/test/queries/clientpositive/groupby1.q ql/src/test/queries/clientpositive/groupby1.q index a8c9a8dcf8..cd3a12b44e 100755 --- ql/src/test/queries/clientpositive/groupby1.q +++ ql/src/test/queries/clientpositive/groupby1.q @@ -1,3 +1,5 @@ +-- due to testMTQueries1 +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; set hive.explain.user=false; set hive.map.aggr=false; diff --git ql/src/test/queries/clientpositive/groupby1_limit.q ql/src/test/queries/clientpositive/groupby1_limit.q index b8e389e511..6c40e19540 100644 --- ql/src/test/queries/clientpositive/groupby1_limit.q +++ ql/src/test/queries/clientpositive/groupby1_limit.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; set mapred.reduce.tasks=31; diff --git ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q index 2b799f87eb..40976ee707 100644 --- ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q +++ ql/src/test/queries/clientpositive/groupby_multi_single_reducer.q @@ -1,3 +1,6 @@ +set hive.stats.column.autogather=false; +-- due to L137 in LimitPushDownOptimization Not safe to continue for RS-GBY-GBY-LIM kind of pipelines. See HIVE-10607 for more. + set hive.multigroupby.singlereducer=true; -- SORT_QUERY_RESULTS diff --git ql/src/test/queries/clientpositive/hybridgrace_hashjoin_1.q ql/src/test/queries/clientpositive/hybridgrace_hashjoin_1.q index 9c19a8678c..ec34e7a555 100644 --- ql/src/test/queries/clientpositive/hybridgrace_hashjoin_1.q +++ ql/src/test/queries/clientpositive/hybridgrace_hashjoin_1.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; set hive.explain.user=false; -- Hybrid Grace Hash Join diff --git ql/src/test/queries/clientpositive/infer_bucket_sort_convert_join.q ql/src/test/queries/clientpositive/infer_bucket_sort_convert_join.q index 6809b721be..e4170283f3 100644 --- ql/src/test/queries/clientpositive/infer_bucket_sort_convert_join.q +++ ql/src/test/queries/clientpositive/infer_bucket_sort_convert_join.q @@ -1,3 +1,12 @@ +set hive.stats.column.autogather=false; +-- sounds weird: +-- on master, when auto=true, hive.mapjoin.localtask.max.memory.usage will be 0.55 as there is a gby +-- L132 of LocalMapJoinProcFactory +-- when execute in CLI, hive.exec.submit.local.task.via.child is true and we can see the error +-- if set hive.exec.submit.local.task.via.child=false, we can see it. +-- with patch, we just merge the tasks. hive.exec.submit.local.task.via.child=false due to pom.xml setting +-- however, even after change it to true, it still fails. + set hive.mapred.mode=nonstrict; set hive.exec.infer.bucket.sort=true; set hive.exec.infer.bucket.sort.num.buckets.power.two=true; diff --git ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q index 6824c1c032..c0ddb8bce6 100644 --- ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q +++ ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; set hive.exec.infer.bucket.sort=true; set hive.exec.infer.bucket.sort.num.buckets.power.two=true; diff --git ql/src/test/queries/clientpositive/input11_limit.q ql/src/test/queries/clientpositive/input11_limit.q index 052a72ee68..211c37adc5 100644 --- ql/src/test/queries/clientpositive/input11_limit.q +++ ql/src/test/queries/clientpositive/input11_limit.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; -- SORT_QUERY_RESULTS diff --git ql/src/test/queries/clientpositive/input14_limit.q ql/src/test/queries/clientpositive/input14_limit.q index 7316752a6d..2f6e4e47c9 100644 --- ql/src/test/queries/clientpositive/input14_limit.q +++ ql/src/test/queries/clientpositive/input14_limit.q @@ -1,3 +1,5 @@ +set hive.stats.column.autogather=false; + CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN diff --git ql/src/test/queries/clientpositive/join2.q ql/src/test/queries/clientpositive/join2.q index 8aedd561e2..c3c7c241e9 100644 --- ql/src/test/queries/clientpositive/join2.q +++ ql/src/test/queries/clientpositive/join2.q @@ -1,3 +1,5 @@ +-- due to testMTQueries1 +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; -- SORT_QUERY_RESULTS diff --git ql/src/test/queries/clientpositive/metadata_only_queries.q ql/src/test/queries/clientpositive/metadata_only_queries.q index 8581a46b2d..bcf320b0c5 100644 --- ql/src/test/queries/clientpositive/metadata_only_queries.q +++ ql/src/test/queries/clientpositive/metadata_only_queries.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; set hive.explain.user=false; set hive.compute.query.using.stats=true; diff --git ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q index 1af813e3ed..692c414354 100644 --- ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q +++ ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.stats.dbclass=fs; set hive.compute.query.using.stats=true; set hive.explain.user=false; diff --git ql/src/test/queries/clientpositive/multiMapJoin1.q ql/src/test/queries/clientpositive/multiMapJoin1.q index 5c49b4c64f..6e16af4617 100644 --- ql/src/test/queries/clientpositive/multiMapJoin1.q +++ ql/src/test/queries/clientpositive/multiMapJoin1.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,org.apache.hadoop.hive.ql.hooks.PrintCompletedTasksHook; diff --git ql/src/test/queries/clientpositive/orc_wide_table.q ql/src/test/queries/clientpositive/orc_wide_table.q index 422a3c24b1..d2ec3857d0 100644 --- ql/src/test/queries/clientpositive/orc_wide_table.q +++ ql/src/test/queries/clientpositive/orc_wide_table.q @@ -1,4 +1,5 @@ set hive.mapred.mode=nonstrict; +set hive.stats.column.autogather=false; drop table if exists test_txt; drop table if exists test_orc; create table test_txt( diff --git ql/src/test/queries/clientpositive/partition_coltype_literals.q ql/src/test/queries/clientpositive/partition_coltype_literals.q index eb56b1a93d..8da4876b70 100644 --- ql/src/test/queries/clientpositive/partition_coltype_literals.q +++ ql/src/test/queries/clientpositive/partition_coltype_literals.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.compute.query.using.stats=false; drop table if exists partcoltypenum; create table partcoltypenum (key int, value string) partitioned by (tint tinyint, sint smallint, bint bigint); diff --git ql/src/test/queries/clientpositive/row__id.q ql/src/test/queries/clientpositive/row__id.q index d9cb7b0ff6..6aaa40f68f 100644 --- ql/src/test/queries/clientpositive/row__id.q +++ ql/src/test/queries/clientpositive/row__id.q @@ -1,3 +1,5 @@ +-- tid is flaky when compute column stats +set hive.stats.column.autogather=false; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; drop table if exists hello_acid; diff --git ql/src/test/queries/clientpositive/smb_join_partition_key.q ql/src/test/queries/clientpositive/smb_join_partition_key.q index 160bf5e36a..23027f8aa5 100644 --- ql/src/test/queries/clientpositive/smb_join_partition_key.q +++ ql/src/test/queries/clientpositive/smb_join_partition_key.q @@ -1,3 +1,5 @@ +--because p1 is decimal, in derby, when it retrieves partition with decimal, it will use partval = 100.0, rather than 100. As a result, the partition will not be found and it throws exception. +set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; SET hive.enforce.sortmergebucketmapjoin=false; SET hive.auto.convert.sortmerge.join=true; diff --git ql/src/test/queries/clientpositive/udf_round_2.q ql/src/test/queries/clientpositive/udf_round_2.q index 43988c1225..38885a97d4 100644 --- ql/src/test/queries/clientpositive/udf_round_2.q +++ ql/src/test/queries/clientpositive/udf_round_2.q @@ -1,4 +1,5 @@ set hive.fetch.task.conversion=more; +set hive.stats.column.autogather=false; -- test for NaN (not-a-number) create table tstTbl1(n double); diff --git ql/src/test/queries/clientpositive/vector_groupby_rollup1.q ql/src/test/queries/clientpositive/vector_groupby_rollup1.q index e08f8b9393..c2441c6864 100644 --- ql/src/test/queries/clientpositive/vector_groupby_rollup1.q +++ ql/src/test/queries/clientpositive/vector_groupby_rollup1.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; SET hive.vectorized.execution.reduce.enabled=true; diff --git ql/src/test/queries/clientpositive/vector_multi_insert.q ql/src/test/queries/clientpositive/vector_multi_insert.q index c56ee1c4aa..e6bfb96794 100644 --- ql/src/test/queries/clientpositive/vector_multi_insert.q +++ ql/src/test/queries/clientpositive/vector_multi_insert.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; diff --git ql/src/test/queries/clientpositive/vector_udf_character_length.q ql/src/test/queries/clientpositive/vector_udf_character_length.q index 19a5260ddc..e49a091b34 100644 --- ql/src/test/queries/clientpositive/vector_udf_character_length.q +++ ql/src/test/queries/clientpositive/vector_udf_character_length.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; diff --git ql/src/test/queries/clientpositive/vector_udf_octet_length.q ql/src/test/queries/clientpositive/vector_udf_octet_length.q index 06a49852a2..af4c7c4a7f 100644 --- ql/src/test/queries/clientpositive/vector_udf_octet_length.q +++ ql/src/test/queries/clientpositive/vector_udf_octet_length.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; diff --git ql/src/test/queries/clientpositive/vector_varchar_4.q ql/src/test/queries/clientpositive/vector_varchar_4.q index 80f84d8b9f..b3402d0df2 100644 --- ql/src/test/queries/clientpositive/vector_varchar_4.q +++ ql/src/test/queries/clientpositive/vector_varchar_4.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; @@ -50,4 +51,4 @@ insert overwrite table varchar_lazy_binary_columnar select t, si, i, b, f, d, s -- insert overwrite table varchar_lazy_binary_columnar select t, si, i, b, f, d, s from vectortab2korc; --- select count(*) as cnt from varchar_lazy_binary_columnar group by vs order by cnt asc; \ No newline at end of file +-- select count(*) as cnt from varchar_lazy_binary_columnar group by vs order by cnt asc; diff --git ql/src/test/queries/clientpositive/vector_varchar_simple.q ql/src/test/queries/clientpositive/vector_varchar_simple.q index 6f753a748d..352ec3aebc 100644 --- ql/src/test/queries/clientpositive/vector_varchar_simple.q +++ ql/src/test/queries/clientpositive/vector_varchar_simple.q @@ -1,3 +1,4 @@ +set hive.stats.column.autogather=false; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; diff --git ql/src/test/results/clientpositive/acid_table_stats.q.out ql/src/test/results/clientpositive/acid_table_stats.q.out index 6ab6b4346d..15f083bd8a 100644 --- ql/src/test/results/clientpositive/acid_table_stats.q.out +++ ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -95,6 +95,7 @@ Database: default Table: acid #### A masked pattern was here #### Partition Parameters: + COLUMN_STATS_ACCURATE {} numFiles 2 numRows 0 rawDataSize 0 @@ -232,11 +233,15 @@ PREHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@acid PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid +PREHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@acid POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid +POSTHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: desc formatted acid partition(ds='2008-04-08') PREHOOK: type: DESCTABLE @@ -514,11 +519,15 @@ PREHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@acid PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid +PREHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@acid POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid +POSTHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: explain select max(key) from acid where ds='2008-04-08' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/alterColumnStatsPart.q.out ql/src/test/results/clientpositive/alterColumnStatsPart.q.out index a315a6be39..fc5717b171 100644 --- ql/src/test/results/clientpositive/alterColumnStatsPart.q.out +++ ql/src/test/results/clientpositive/alterColumnStatsPart.q.out @@ -159,11 +159,15 @@ PREHOOK: query: analyze table p partition(c=1) compute statistics for columns a PREHOOK: type: QUERY PREHOOK: Input: default@p PREHOOK: Input: default@p@c=1 +PREHOOK: Output: default@p +PREHOOK: Output: default@p@c=1 #### A masked pattern was here #### POSTHOOK: query: analyze table p partition(c=1) compute statistics for columns a POSTHOOK: type: QUERY POSTHOOK: Input: default@p POSTHOOK: Input: default@p@c=1 +POSTHOOK: Output: default@p +POSTHOOK: Output: default@p@c=1 #### A masked pattern was here #### PREHOOK: query: explain select max(a) from p where c=1 PREHOOK: type: QUERY @@ -380,11 +384,15 @@ PREHOOK: query: analyze table p partition(c=100) compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@p PREHOOK: Input: default@p@c=100 +PREHOOK: Output: default@p +PREHOOK: Output: default@p@c=100 #### A masked pattern was here #### POSTHOOK: query: analyze table p partition(c=100) compute statistics for columns a POSTHOOK: type: QUERY POSTHOOK: Input: default@p POSTHOOK: Input: default@p@c=100 +POSTHOOK: Output: default@p +POSTHOOK: Output: default@p@c=100 #### A masked pattern was here #### PREHOOK: query: explain select max(a) from p where c=100 PREHOOK: type: QUERY @@ -485,11 +493,15 @@ PREHOOK: query: analyze table p partition(c=100) compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@p PREHOOK: Input: default@p@c=100 +PREHOOK: Output: default@p +PREHOOK: Output: default@p@c=100 #### A masked pattern was here #### POSTHOOK: query: analyze table p partition(c=100) compute statistics for columns a POSTHOOK: type: QUERY POSTHOOK: Input: default@p POSTHOOK: Input: default@p@c=100 +POSTHOOK: Output: default@p +POSTHOOK: Output: default@p@c=100 #### A masked pattern was here #### PREHOOK: query: explain select max(a) from p where c=100 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/alter_partition_update_status.q.out ql/src/test/results/clientpositive/alter_partition_update_status.q.out index caa1f8dd32..15aab1e64c 100644 --- ql/src/test/results/clientpositive/alter_partition_update_status.q.out +++ ql/src/test/results/clientpositive/alter_partition_update_status.q.out @@ -22,11 +22,15 @@ PREHOOK: query: ANALYZE TABLE src_stat_part_one PARTITION(partitionId=1) COMPUTE PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part_one PREHOOK: Input: default@src_stat_part_one@partitionid=1 +PREHOOK: Output: default@src_stat_part_one +PREHOOK: Output: default@src_stat_part_one@partitionid=1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part_one PARTITION(partitionId=1) COMPUTE STATISTICS for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part_one POSTHOOK: Input: default@src_stat_part_one@partitionid=1 +POSTHOOK: Output: default@src_stat_part_one +POSTHOOK: Output: default@src_stat_part_one@partitionid=1 #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part_one PARTITION(partitionId=1) key PREHOOK: type: DESCTABLE @@ -74,11 +78,15 @@ PREHOOK: query: ANALYZE TABLE src_stat_part_two PARTITION(px=1) COMPUTE STATISTI PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part_two PREHOOK: Input: default@src_stat_part_two@px=1/py=a +PREHOOK: Output: default@src_stat_part_two +PREHOOK: Output: default@src_stat_part_two@px=1/py=a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part_two PARTITION(px=1) COMPUTE STATISTICS for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part_two POSTHOOK: Input: default@src_stat_part_two@px=1/py=a +POSTHOOK: Output: default@src_stat_part_two +POSTHOOK: Output: default@src_stat_part_two@px=1/py=a #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part_two PARTITION(px=1, py='a') key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/alter_table_column_stats.q.out ql/src/test/results/clientpositive/alter_table_column_stats.q.out index b93e9de1c1..f87d9e52df 100644 --- ql/src/test/results/clientpositive/alter_table_column_stats.q.out +++ ql/src/test/results/clientpositive/alter_table_column_stats.q.out @@ -77,10 +77,12 @@ PREHOOK: query: analyze table testtable0 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable0 POSTHOOK: query: analyze table testtable0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable0 PREHOOK: query: describe formatted statsdb1.testtable0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable0 @@ -455,12 +457,18 @@ PREHOOK: Input: statsdb1@testpart0 PREHOOK: Input: statsdb1@testpart0@part=part1 PREHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testpart0 +PREHOOK: Output: statsdb1@testpart0@part=part1 +PREHOOK: Output: statsdb1@testpart0@part=part2 POSTHOOK: query: analyze table testpart0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testpart0 POSTHOOK: Input: statsdb1@testpart0@part=part1 POSTHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testpart0 +POSTHOOK: Output: statsdb1@testpart0@part=part1 +POSTHOOK: Output: statsdb1@testpart0@part=part2 PREHOOK: query: describe formatted statsdb1.testpart0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testpart0 @@ -1617,10 +1625,12 @@ PREHOOK: query: analyze table testtable0 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable0 POSTHOOK: query: analyze table testtable0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable0 PREHOOK: query: describe formatted statsdb1.testtable0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable0 @@ -1995,12 +2005,18 @@ PREHOOK: Input: statsdb1@testpart0 PREHOOK: Input: statsdb1@testpart0@part=part1 PREHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testpart0 +PREHOOK: Output: statsdb1@testpart0@part=part1 +PREHOOK: Output: statsdb1@testpart0@part=part2 POSTHOOK: query: analyze table testpart0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testpart0 POSTHOOK: Input: statsdb1@testpart0@part=part1 POSTHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testpart0 +POSTHOOK: Output: statsdb1@testpart0@part=part1 +POSTHOOK: Output: statsdb1@testpart0@part=part2 PREHOOK: query: describe formatted statsdb1.testpart0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testpart0 diff --git ql/src/test/results/clientpositive/alter_table_update_status.q.out ql/src/test/results/clientpositive/alter_table_update_status.q.out index 452f37bba3..9e9164f144 100644 --- ql/src/test/results/clientpositive/alter_table_update_status.q.out +++ ql/src/test/results/clientpositive/alter_table_update_status.q.out @@ -35,10 +35,12 @@ POSTHOOK: Output: default@src_stat_int PREHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat +PREHOOK: Output: default@src_stat #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat +POSTHOOK: Output: default@src_stat #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat key PREHOOK: type: DESCTABLE @@ -81,10 +83,12 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true PREHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_int +PREHOOK: Output: default@src_stat_int #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_int +POSTHOOK: Output: default@src_stat_int #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_int key PREHOOK: type: DESCTABLE @@ -95,7 +99,7 @@ POSTHOOK: Input: default@src_stat_int # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector key double 66.0 406.0 10 15 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') PREHOOK: type: ALTERTABLE_UPDATETABLESTATS POSTHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') @@ -109,7 +113,7 @@ POSTHOOK: Input: default@src_stat_int # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector key double 333.22 22.22 10 2222 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: create database if not exists dummydb PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:dummydb diff --git ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out index 1dcc1fcf4e..ef9860ef99 100644 --- ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out +++ ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out @@ -35,10 +35,12 @@ POSTHOOK: Output: default@src_stat_int PREHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat +PREHOOK: Output: default@src_stat #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat +POSTHOOK: Output: default@src_stat #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat key PREHOOK: type: DESCTABLE @@ -81,10 +83,12 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true PREHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_int +PREHOOK: Output: default@src_stat_int #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_int +POSTHOOK: Output: default@src_stat_int #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_int key PREHOOK: type: DESCTABLE @@ -95,7 +99,7 @@ POSTHOOK: Input: default@src_stat_int # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector key double 66.0 406.0 10 15 from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') PREHOOK: type: ALTERTABLE_UPDATETABLESTATS POSTHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') @@ -109,7 +113,7 @@ POSTHOOK: Input: default@src_stat_int # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector key double 333.22 22.22 10 2222 from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: create database if not exists dummydb PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:dummydb diff --git ql/src/test/results/clientpositive/analyze_tbl_date.q.out ql/src/test/results/clientpositive/analyze_tbl_date.q.out index a0cdbca8c1..eac13cc1bc 100644 --- ql/src/test/results/clientpositive/analyze_tbl_date.q.out +++ ql/src/test/results/clientpositive/analyze_tbl_date.q.out @@ -16,10 +16,12 @@ POSTHOOK: Lineage: test_table.d EXPRESSION [(values__tmp__table__1)values__tmp__ PREHOOK: query: analyze table test_table compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@test_table +PREHOOK: Output: default@test_table #### A masked pattern was here #### POSTHOOK: query: analyze table test_table compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table +POSTHOOK: Output: default@test_table #### A masked pattern was here #### PREHOOK: query: describe formatted test_table PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/analyze_tbl_part.q.out ql/src/test/results/clientpositive/analyze_tbl_part.q.out index a7a2220304..ec18625127 100644 --- ql/src/test/results/clientpositive/analyze_tbl_part.q.out +++ ql/src/test/results/clientpositive/analyze_tbl_part.q.out @@ -35,12 +35,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part PREHOOK: Input: default@src_stat_part@partitionid=1 PREHOOK: Input: default@src_stat_part@partitionid=2 +PREHOOK: Output: default@src_stat_part +PREHOOK: Output: default@src_stat_part@partitionid=1 +PREHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part partition (partitionId) COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part POSTHOOK: Input: default@src_stat_part@partitionid=1 POSTHOOK: Input: default@src_stat_part@partitionid=2 +POSTHOOK: Output: default@src_stat_part +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) key PREHOOK: type: DESCTABLE @@ -56,12 +62,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part PREHOOK: Input: default@src_stat_part@partitionid=1 PREHOOK: Input: default@src_stat_part@partitionid=2 +PREHOOK: Output: default@src_stat_part +PREHOOK: Output: default@src_stat_part@partitionid=1 +PREHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part partition (partitionId) COMPUTE STATISTICS for columns key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part POSTHOOK: Input: default@src_stat_part@partitionid=1 POSTHOOK: Input: default@src_stat_part@partitionid=2 +POSTHOOK: Output: default@src_stat_part +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) key PREHOOK: type: DESCTABLE @@ -117,19 +129,27 @@ PREHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\' PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_string_part PREHOOK: Input: default@src_stat_string_part@partitionname=p%271 +PREHOOK: Output: default@src_stat_string_part +PREHOOK: Output: default@src_stat_string_part@partitionname=p%271 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\'1") COMPUTE STATISTICS for columns key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_string_part POSTHOOK: Input: default@src_stat_string_part@partitionname=p%271 +POSTHOOK: Output: default@src_stat_string_part +POSTHOOK: Output: default@src_stat_string_part@partitionname=p%271 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\"1") COMPUTE STATISTICS for columns key, value PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_string_part PREHOOK: Input: default@src_stat_string_part@partitionname=p%221 +PREHOOK: Output: default@src_stat_string_part +PREHOOK: Output: default@src_stat_string_part@partitionname=p%221 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\"1") COMPUTE STATISTICS for columns key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_string_part POSTHOOK: Input: default@src_stat_string_part@partitionname=p%221 +POSTHOOK: Output: default@src_stat_string_part +POSTHOOK: Output: default@src_stat_string_part@partitionname=p%221 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out index 6e2975e671..d81f5660ea 100644 --- ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out +++ ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out @@ -59,10 +59,12 @@ POSTHOOK: Output: default@over1k PREHOOK: query: analyze table over1k compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@over1k +PREHOOK: Output: default@over1k #### A masked pattern was here #### POSTHOOK: query: analyze table over1k compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@over1k +POSTHOOK: Output: default@over1k #### A masked pattern was here #### PREHOOK: query: explain select count(*) from over1k where ( (t=1 and si=2) diff --git ql/src/test/results/clientpositive/annotate_stats_filter.q.out ql/src/test/results/clientpositive/annotate_stats_filter.q.out index b2f983606e..bf139ccd77 100644 --- ql/src/test/results/clientpositive/annotate_stats_filter.q.out +++ ql/src/test/results/clientpositive/annotate_stats_filter.q.out @@ -112,10 +112,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select * from loc_orc where state='OH' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_groupby.q.out ql/src/test/results/clientpositive/annotate_stats_groupby.q.out index f9a1eb8312..0b26771f4f 100644 --- ql/src/test/results/clientpositive/annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/annotate_stats_groupby.q.out @@ -76,10 +76,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select a, c, min(b) from ( select state as a, locid as b, count(*) as c @@ -177,10 +179,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select year from loc_orc group by year PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out index 31c4ed147f..d95af92acd 100644 --- ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out +++ ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out @@ -29,10 +29,12 @@ POSTHOOK: Output: default@location PREHOOK: query: analyze table location compute statistics for columns state, country PREHOOK: type: QUERY PREHOOK: Input: default@location +PREHOOK: Output: default@location #### A masked pattern was here #### POSTHOOK: query: analyze table location compute statistics for columns state, country POSTHOOK: type: QUERY POSTHOOK: Input: default@location +POSTHOOK: Output: default@location #### A masked pattern was here #### PREHOOK: query: explain select state, country from location group by state, country PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_join.q.out ql/src/test/results/clientpositive/annotate_stats_join.q.out index a7e73a06c7..9173e7acf1 100644 --- ql/src/test/results/clientpositive/annotate_stats_join.q.out +++ ql/src/test/results/clientpositive/annotate_stats_join.q.out @@ -97,26 +97,32 @@ POSTHOOK: Output: default@loc PREHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid PREHOOK: type: QUERY PREHOOK: Input: default@emp +PREHOOK: Output: default@emp #### A masked pattern was here #### POSTHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp +POSTHOOK: Output: default@emp #### A masked pattern was here #### PREHOOK: query: analyze table dept compute statistics for columns deptname,deptid PREHOOK: type: QUERY PREHOOK: Input: default@dept +PREHOOK: Output: default@dept #### A masked pattern was here #### POSTHOOK: query: analyze table dept compute statistics for columns deptname,deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@dept +POSTHOOK: Output: default@dept #### A masked pattern was here #### PREHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc +PREHOOK: Output: default@loc #### A masked pattern was here #### POSTHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc +POSTHOOK: Output: default@loc #### A masked pattern was here #### PREHOOK: query: explain select * from emp e join dept d on (e.deptid = d.deptid) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out index e04c1c6bc5..7d3c0abc61 100644 --- ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out +++ ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out @@ -297,10 +297,12 @@ POSTHOOK: Output: default@store PREHOOK: query: analyze table store compute statistics for columns s_store_sk, s_floor_space PREHOOK: type: QUERY PREHOOK: Input: default@store +PREHOOK: Output: default@store #### A masked pattern was here #### POSTHOOK: query: analyze table store compute statistics for columns s_store_sk, s_floor_space POSTHOOK: type: QUERY POSTHOOK: Input: default@store +POSTHOOK: Output: default@store #### A masked pattern was here #### PREHOOK: query: analyze table store_bigint compute statistics PREHOOK: type: QUERY @@ -313,10 +315,12 @@ POSTHOOK: Output: default@store_bigint PREHOOK: query: analyze table store_bigint compute statistics for columns s_store_sk, s_floor_space PREHOOK: type: QUERY PREHOOK: Input: default@store_bigint +PREHOOK: Output: default@store_bigint #### A masked pattern was here #### POSTHOOK: query: analyze table store_bigint compute statistics for columns s_store_sk, s_floor_space POSTHOOK: type: QUERY POSTHOOK: Input: default@store_bigint +POSTHOOK: Output: default@store_bigint #### A masked pattern was here #### PREHOOK: query: analyze table store_sales compute statistics PREHOOK: type: QUERY @@ -329,10 +333,12 @@ POSTHOOK: Output: default@store_sales PREHOOK: query: analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity PREHOOK: type: QUERY PREHOOK: Input: default@store_sales +PREHOOK: Output: default@store_sales #### A masked pattern was here #### POSTHOOK: query: analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity POSTHOOK: type: QUERY POSTHOOK: Input: default@store_sales +POSTHOOK: Output: default@store_sales #### A masked pattern was here #### PREHOOK: query: analyze table customer_address compute statistics PREHOOK: type: QUERY @@ -345,10 +351,12 @@ POSTHOOK: Output: default@customer_address PREHOOK: query: analyze table customer_address compute statistics for columns ca_address_sk PREHOOK: type: QUERY PREHOOK: Input: default@customer_address +PREHOOK: Output: default@customer_address #### A masked pattern was here #### POSTHOOK: query: analyze table customer_address compute statistics for columns ca_address_sk POSTHOOK: type: QUERY POSTHOOK: Input: default@customer_address +POSTHOOK: Output: default@customer_address #### A masked pattern was here #### PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_limit.q.out ql/src/test/results/clientpositive/annotate_stats_limit.q.out index ea181cb8f6..5139db4d64 100644 --- ql/src/test/results/clientpositive/annotate_stats_limit.q.out +++ ql/src/test/results/clientpositive/annotate_stats_limit.q.out @@ -55,10 +55,12 @@ POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name PREHOOK: query: analyze table loc_orc compute statistics for columns state, locid, zip, year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state, locid, zip, year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select * from loc_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_part.q.out ql/src/test/results/clientpositive/annotate_stats_part.q.out index def4d4faee..3a94a6a4e3 100644 --- ql/src/test/results/clientpositive/annotate_stats_part.q.out +++ ql/src/test/results/clientpositive/annotate_stats_part.q.out @@ -273,11 +273,15 @@ PREHOOK: query: analyze table loc_orc partition(year='2001') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc PREHOOK: Input: default@loc_orc@year=2001 +PREHOOK: Output: default@loc_orc +PREHOOK: Output: default@loc_orc@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc partition(year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc POSTHOOK: Input: default@loc_orc@year=2001 +POSTHOOK: Output: default@loc_orc +POSTHOOK: Output: default@loc_orc@year=2001 #### A masked pattern was here #### PREHOOK: query: explain select zip from loc_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_select.q.out ql/src/test/results/clientpositive/annotate_stats_select.q.out index dec7f402ad..7f402db8b3 100644 --- ql/src/test/results/clientpositive/annotate_stats_select.q.out +++ ql/src/test/results/clientpositive/annotate_stats_select.q.out @@ -113,10 +113,12 @@ STAGE PLANS: PREHOOK: query: analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1 PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_orc +PREHOOK: Output: default@alltypes_orc #### A masked pattern was here #### POSTHOOK: query: analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_orc +POSTHOOK: Output: default@alltypes_orc #### A masked pattern was here #### PREHOOK: query: explain select * from alltypes_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_table.q.out ql/src/test/results/clientpositive/annotate_stats_table.q.out index 5d443f1381..e93ce4f36d 100644 --- ql/src/test/results/clientpositive/annotate_stats_table.q.out +++ ql/src/test/results/clientpositive/annotate_stats_table.q.out @@ -120,10 +120,12 @@ STAGE PLANS: PREHOOK: query: analyze table emp_orc compute statistics for columns deptid PREHOOK: type: QUERY PREHOOK: Input: default@emp_orc +PREHOOK: Output: default@emp_orc #### A masked pattern was here #### POSTHOOK: query: analyze table emp_orc compute statistics for columns deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_orc +POSTHOOK: Output: default@emp_orc #### A masked pattern was here #### PREHOOK: query: explain select * from emp_orc PREHOOK: type: QUERY @@ -170,10 +172,12 @@ STAGE PLANS: PREHOOK: query: analyze table emp_orc compute statistics for columns lastname,deptid PREHOOK: type: QUERY PREHOOK: Input: default@emp_orc +PREHOOK: Output: default@emp_orc #### A masked pattern was here #### POSTHOOK: query: analyze table emp_orc compute statistics for columns lastname,deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_orc +POSTHOOK: Output: default@emp_orc #### A masked pattern was here #### PREHOOK: query: explain select * from emp_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_union.q.out ql/src/test/results/clientpositive/annotate_stats_union.q.out index 059f261d97..3b4b169dc8 100644 --- ql/src/test/results/clientpositive/annotate_stats_union.q.out +++ ql/src/test/results/clientpositive/annotate_stats_union.q.out @@ -55,10 +55,12 @@ POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select state from loc_orc PREHOOK: type: QUERY @@ -282,18 +284,22 @@ PREHOOK: query: analyze table loc_staging compute statistics for columns state,l PREHOOK: type: QUERY PREHOOK: Input: test@loc_staging #### A masked pattern was here #### +PREHOOK: Output: test@loc_staging POSTHOOK: query: analyze table loc_staging compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: test@loc_staging #### A masked pattern was here #### +POSTHOOK: Output: test@loc_staging PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: test@loc_orc #### A masked pattern was here #### +PREHOOK: Output: test@loc_orc POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: test@loc_orc #### A masked pattern was here #### +POSTHOOK: Output: test@loc_orc PREHOOK: query: explain select * from (select state from default.loc_orc union all select state from test.loc_orc) temp PREHOOK: type: QUERY POSTHOOK: query: explain select * from (select state from default.loc_orc union all select state from test.loc_orc) temp diff --git ql/src/test/results/clientpositive/autoColumnStats_1.q.out ql/src/test/results/clientpositive/autoColumnStats_1.q.out index 4cf6df18f9..5f78d88dc2 100644 --- ql/src/test/results/clientpositive/autoColumnStats_1.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_1.q.out @@ -382,6 +382,46 @@ POSTHOOK: query: create table if not exists nzhang_part14 (key string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: desc formatted nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment + +key string + +# Partition Information +# col_name data_type comment + +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a @@ -407,6 +447,44 @@ POSTHOOK: Output: default@nzhang_part14@value= POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ POSTHOOK: Lineage: nzhang_part14 PARTITION(value= ).key EXPRESSION [] POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] +PREHOOK: query: desc formatted nzhang_part14 partition (value=' ') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 partition (value=' ') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment + +key string + +# Partition Information +# col_name data_type comment + +value string + +# Detailed Partition Information +Partition Value: [ ] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 4 + totalSize 6 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain select key from nzhang_part14 PREHOOK: type: QUERY POSTHOOK: query: explain select key from nzhang_part14 @@ -477,11 +555,11 @@ STAGE PLANS: Processor Tree: TableScan alias: nzhang_part14 - Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) @@ -958,6 +1036,12 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/autoColumnStats_10.q.out ql/src/test/results/clientpositive/autoColumnStats_10.q.out new file mode 100644 index 0000000000..6b730c2bfd --- /dev/null +++ ql/src/test/results/clientpositive/autoColumnStats_10.q.out @@ -0,0 +1,452 @@ +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int 1 1 0 1 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int 1 2 0 2 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} diff --git ql/src/test/results/clientpositive/autoColumnStats_2.q.out ql/src/test/results/clientpositive/autoColumnStats_2.q.out index 791e6ae2fd..169d0f04c3 100644 --- ql/src/test/results/clientpositive/autoColumnStats_2.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_2.q.out @@ -125,18 +125,20 @@ PREHOOK: Input: default@a POSTHOOK: query: describe formatted a key POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@a -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -key string 0 205 2.812 3 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +key string 0 309 2.812 3 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: describe formatted b key PREHOOK: type: DESCTABLE PREHOOK: Input: default@b POSTHOOK: query: describe formatted b key POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -key string 0 205 2.812 3 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +key string 0 309 2.812 3 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: from src insert overwrite table a select * insert into table b select * @@ -231,18 +233,20 @@ PREHOOK: Input: default@b POSTHOOK: query: describe formatted b key POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -key string 0 205 2.812 3 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +key string 0 309 2.812 3 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: describe formatted b value PREHOOK: type: DESCTABLE PREHOOK: Input: default@b POSTHOOK: query: describe formatted b value POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -value string 0 214 6.812 7 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +value string 0 309 6.812 7 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: insert into table b select NULL, NULL from src limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -251,26 +255,28 @@ POSTHOOK: query: insert into table b select NULL, NULL from src limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@b -POSTHOOK: Lineage: b.key SIMPLE [] -POSTHOOK: Lineage: b.value SIMPLE [] +POSTHOOK: Lineage: b.key EXPRESSION [] +POSTHOOK: Lineage: b.value EXPRESSION [] PREHOOK: query: describe formatted b key PREHOOK: type: DESCTABLE PREHOOK: Input: default@b POSTHOOK: query: describe formatted b key POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -key string 10 205 2.812 3 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +key string 10 309 2.812 3 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: describe formatted b value PREHOOK: type: DESCTABLE PREHOOK: Input: default@b POSTHOOK: query: describe formatted b value POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -value string 10 214 6.812 7 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +value string 10 309 6.812 7 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: insert into table b(value) select key+100000 from src limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -287,18 +293,20 @@ PREHOOK: Input: default@b POSTHOOK: query: describe formatted b key POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -key string 20 205 2.812 3 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +key string 20 309 2.812 3 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: describe formatted b value PREHOOK: type: DESCTABLE PREHOOK: Input: default@b POSTHOOK: query: describe formatted b value POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@b -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment - -value string 10 214 8.0 8 from deserializer +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +value string 10 319 8.0 8 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} PREHOOK: query: drop table src_multi2 PREHOOK: type: DROPTABLE POSTHOOK: query: drop table src_multi2 @@ -467,11 +475,11 @@ STAGE PLANS: Processor Tree: TableScan alias: nzhang_part14 - Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: drop table alter5 @@ -514,7 +522,7 @@ Database: default Table: alter5 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\"}} numFiles 0 numRows 0 rawDataSize 0 @@ -687,7 +695,6 @@ Database: default Table: alter5 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"col1\":\"true\"}} numFiles 1 totalSize 1906 #### A masked pattern was here #### @@ -1079,6 +1086,12 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/autoColumnStats_3.q.out ql/src/test/results/clientpositive/autoColumnStats_3.q.out index 2f70095b7a..3a23c94b13 100644 --- ql/src/test/results/clientpositive/autoColumnStats_3.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_3.q.out @@ -13,10 +13,12 @@ POSTHOOK: Output: default@src_multi1 PREHOOK: query: analyze table src_multi1 compute statistics for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_multi1 +PREHOOK: Output: default@src_multi1 #### A masked pattern was here #### POSTHOOK: query: analyze table src_multi1 compute statistics for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_multi1 +POSTHOOK: Output: default@src_multi1 #### A masked pattern was here #### PREHOOK: query: describe formatted src_multi1 PREHOOK: type: DESCTABLE @@ -228,11 +230,15 @@ PREHOOK: query: analyze table nzhang_part14 partition(ds='1', hr='3') compute st PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_part14 PREHOOK: Input: default@nzhang_part14@ds=1/hr=3 +PREHOOK: Output: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14@ds=1/hr=3 #### A masked pattern was here #### POSTHOOK: query: analyze table nzhang_part14 partition(ds='1', hr='3') compute statistics for columns value POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_part14 POSTHOOK: Input: default@nzhang_part14@ds=1/hr=3 +POSTHOOK: Output: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=3 #### A masked pattern was here #### PREHOOK: query: desc formatted nzhang_part14 partition(ds='1', hr='3') PREHOOK: type: DESCTABLE @@ -367,7 +373,7 @@ Database: default Table: nzhang_part14 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} numFiles 2 numRows 4 rawDataSize 12 @@ -407,7 +413,7 @@ Database: default Table: nzhang_part14 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} numFiles 2 numRows 4 rawDataSize 16 diff --git ql/src/test/results/clientpositive/autoColumnStats_4.q.out ql/src/test/results/clientpositive/autoColumnStats_4.q.out index a0581f8d8a..183ef06d4a 100644 --- ql/src/test/results/clientpositive/autoColumnStats_4.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_4.q.out @@ -50,8 +50,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-3, Stage-4 + Stage-3 depends on stages: Stage-0, Stage-4 Stage-4 depends on stages: Stage-2 STAGE PLANS: @@ -138,10 +137,8 @@ STAGE PLANS: name: default.acid_dtt Stage: Stage-3 - Stats-Aggr Operator - - Stage: Stage-5 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b Column Types: int, varchar(128) @@ -197,7 +194,6 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} numFiles 2 numRows 0 rawDataSize 0 diff --git ql/src/test/results/clientpositive/autoColumnStats_5.q.out ql/src/test/results/clientpositive/autoColumnStats_5.q.out index 1298d989a2..36a94df3ef 100644 --- ql/src/test/results/clientpositive/autoColumnStats_5.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_5.q.out @@ -17,7 +17,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -98,10 +97,8 @@ STAGE PLANS: name: default.partitioned1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b Column Types: int, string @@ -255,7 +252,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -336,10 +332,8 @@ STAGE PLANS: name: default.partitioned1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d Column Types: int, string, int, string @@ -449,7 +443,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -530,10 +523,8 @@ STAGE PLANS: name: default.partitioned1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d Column Types: int, string, int, string @@ -603,7 +594,7 @@ Database: default Table: partitioned1 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} numFiles 2 numRows 6 rawDataSize 78 @@ -637,6 +628,6 @@ POSTHOOK: query: desc formatted partitioned1 partition(part=1) c POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@partitioned1 col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitvector -# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector +# col_name data_type comment -c int 100 200 0 2 HL from deserializer +c int from deserializer diff --git ql/src/test/results/clientpositive/autoColumnStats_5a.q.out ql/src/test/results/clientpositive/autoColumnStats_5a.q.out new file mode 100644 index 0000000000..14ead13e96 --- /dev/null +++ ql/src/test/results/clientpositive/autoColumnStats_5a.q.out @@ -0,0 +1,1014 @@ +PREHOOK: query: CREATE TABLE partitioned1(a INT, b STRING) PARTITIONED BY(part INT) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@partitioned1 +POSTHOOK: query: CREATE TABLE partitioned1(a INT, b STRING) PARTITIONED BY(part INT) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@partitioned1 +PREHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(1, 'original') +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(1, 'original') +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: part=1/ + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), UDFToInteger('1') (type: int) + outputColumnNames: a, b, part + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + keys: part (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: struct), _col2 (type: struct) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: Values__Tmp__Table__1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.values__tmp__table__1 + name: default.values__tmp__table__1 + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types struct:struct:int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + part 1 + replace: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-2 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + Column Stats Desc: + Columns: a, b + Column Types: int, string + Table: default.partitioned1 + Is Table Level Stats: false + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert into table partitioned1 partition(part=1) values(1, 'original') +PREHOOK: type: QUERY +PREHOOK: Output: default@partitioned1@part=1 +POSTHOOK: query: insert into table partitioned1 partition(part=1) values(1, 'original') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@partitioned1@part=1 +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).b SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +_col0 _col1 +PREHOOK: query: desc formatted partitioned1 partition(part=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 partition(part=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type comment +# col_name data_type comment + +a int +b string + +# Partition Information +# col_name data_type comment + +part int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: partitioned1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 10 + totalSize 11 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__3 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: part=1/ + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), UDFToInteger('1') (type: int) + outputColumnNames: a, b, part + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + keys: part (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: struct), _col2 (type: struct) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: Values__Tmp__Table__3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"tmp_values_col1":"true","tmp_values_col2":"true"}} + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__3 + numFiles 0 + numRows 0 + rawDataSize 0 + serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 0 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.values__tmp__table__3 + name: default.values__tmp__table__3 + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types struct:struct:int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + part 1 + replace: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-2 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + Column Stats Desc: + Columns: a, b + Column Types: int, string + Table: default.partitioned1 + Is Table Level Stats: false + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +PREHOOK: type: QUERY +PREHOOK: Output: default@partitioned1@part=1 +POSTHOOK: query: insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@partitioned1@part=1 +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).a EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).b SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +_col0 _col1 +PREHOOK: query: explain insert into table partitioned1 partition(part=1) values(1, 'original'),(2, 'original'), (3, 'original'),(4, 'original') +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table partitioned1 partition(part=1) values(1, 'original'),(2, 'original'), (3, 'original'),(4, 'original') +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__5 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), UDFToInteger('1') (type: int) + outputColumnNames: a, b, part + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + keys: part (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct), _col2 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + part 1 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b + Column Types: int, string + Table: default.partitioned1 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: desc formatted partitioned1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type comment +# col_name data_type comment + +a int +b string + +# Partition Information +# col_name data_type comment + +part int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numPartitions 1 + numRows 4 + rawDataSize 40 + totalSize 44 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted partitioned1 partition(part=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 partition(part=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type comment +# col_name data_type comment + +a int +b string + +# Partition Information +# col_name data_type comment + +part int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: partitioned1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + numFiles 2 + numRows 4 + rawDataSize 40 + totalSize 44 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted partitioned1 partition(part=1) a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 partition(part=1) a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitvector +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +a int 1 4 0 4 HL from deserializer diff --git ql/src/test/results/clientpositive/autoColumnStats_6.q.out ql/src/test/results/clientpositive/autoColumnStats_6.q.out index c4ab489b39..70788fde9f 100644 --- ql/src/test/results/clientpositive/autoColumnStats_6.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_6.q.out @@ -30,7 +30,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -113,10 +112,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string diff --git ql/src/test/results/clientpositive/autoColumnStats_7.q.out ql/src/test/results/clientpositive/autoColumnStats_7.q.out index 2dc9fc2d42..acea69e484 100644 --- ql/src/test/results/clientpositive/autoColumnStats_7.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_7.q.out @@ -26,8 +26,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-3, Stage-5 + Stage-3 depends on stages: Stage-0, Stage-5 Stage-4 depends on stages: Stage-2 Stage-5 depends on stages: Stage-4 @@ -112,10 +111,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator - - Stage: Stage-6 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, c1, c2 Column Types: string, int, string diff --git ql/src/test/results/clientpositive/autoColumnStats_8.q.out ql/src/test/results/clientpositive/autoColumnStats_8.q.out index c913d97fe5..3afd07a560 100644 --- ql/src/test/results/clientpositive/autoColumnStats_8.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_8.q.out @@ -48,10 +48,8 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-3, Stage-4, Stage-5 - Stage-7 depends on stages: Stage-3, Stage-4, Stage-5 Stage-1 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-1, Stage-5 Stage-5 depends on stages: Stage-2 STAGE PLANS: @@ -448,25 +446,10 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### - Stage: Stage-6 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.nzhang_part8 - Is Table Level Stats: false - - Stage: Stage-7 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.nzhang_part8 - Is Table Level Stats: false - Stage: Stage-1 Move Operator tables: @@ -496,8 +479,14 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.nzhang_part8 + Is Table Level Stats: false Stage: Stage-5 Map Reduce diff --git ql/src/test/results/clientpositive/autoColumnStats_9.q.out ql/src/test/results/clientpositive/autoColumnStats_9.q.out index fda71e0dd2..78b3316a11 100644 --- ql/src/test/results/clientpositive/autoColumnStats_9.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_9.q.out @@ -20,8 +20,7 @@ STAGE DEPENDENCIES: Stage-7 Stage-5 depends on stages: Stage-7 Stage-0 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-3 Stage-3 depends on stages: Stage-5 STAGE PLANS: @@ -166,10 +165,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string diff --git ql/src/test/results/clientpositive/auto_join1.q.out ql/src/test/results/clientpositive/auto_join1.q.out index 5f4bb7452a..c2d24739ab 100644 --- ql/src/test/results/clientpositive/auto_join1.q.out +++ ql/src/test/results/clientpositive/auto_join1.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index 1dd677c3d6..e67506b3b8 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/auto_join17.q.out ql/src/test/results/clientpositive/auto_join17.q.out index d39c36eac3..0239cf8285 100644 --- ql/src/test/results/clientpositive/auto_join17.q.out +++ ql/src/test/results/clientpositive/auto_join17.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* diff --git ql/src/test/results/clientpositive/auto_join19.q.out ql/src/test/results/clientpositive/auto_join19.q.out index 3f70055d81..e70c5147fb 100644 --- ql/src/test/results/clientpositive/auto_join19.q.out +++ ql/src/test/results/clientpositive/auto_join19.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/auto_join19_inclause.q.out ql/src/test/results/clientpositive/auto_join19_inclause.q.out index 3f70055d81..e70c5147fb 100644 --- ql/src/test/results/clientpositive/auto_join19_inclause.q.out +++ ql/src/test/results/clientpositive/auto_join19_inclause.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/auto_join2.q.out ql/src/test/results/clientpositive/auto_join2.q.out index b17d344985..4132caf7fe 100644 --- ql/src/test/results/clientpositive/auto_join2.q.out +++ ql/src/test/results/clientpositive/auto_join2.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/auto_join25.q.out ql/src/test/results/clientpositive/auto_join25.q.out index 534bdb6ff0..7a83fbc1bf 100644 --- ql/src/test/results/clientpositive/auto_join25.q.out +++ ql/src/test/results/clientpositive/auto_join25.q.out @@ -37,7 +37,7 @@ RUN: Stage-6:CONDITIONAL RUN: Stage-7:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE -RUN: Stage-2:STATS +RUN: Stage-2:COLUMNSTATS PREHOOK: query: SELECT sum(hash(dest1.key,dest1.value)) FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 @@ -80,7 +80,7 @@ RUN: Stage-8:CONDITIONAL RUN: Stage-12:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: SELECT sum(hash(dest_j2.key,dest_j2.value)) FROM dest_j2 PREHOOK: type: QUERY PREHOOK: Input: default@dest_j2 @@ -118,7 +118,7 @@ RUN: Stage-6:CONDITIONAL RUN: Stage-7:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE -RUN: Stage-2:STATS +RUN: Stage-2:COLUMNSTATS PREHOOK: query: SELECT sum(hash(dest_j1.key,dest_j1.value)) FROM dest_j1 PREHOOK: type: QUERY PREHOOK: Input: default@dest_j1 diff --git ql/src/test/results/clientpositive/auto_join26.q.out ql/src/test/results/clientpositive/auto_join26.q.out index e6d966ffad..9b457415e1 100644 --- ql/src/test/results/clientpositive/auto_join26.q.out +++ ql/src/test/results/clientpositive/auto_join26.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/auto_join3.q.out ql/src/test/results/clientpositive/auto_join3.q.out index 35e8273766..5921da093c 100644 --- ql/src/test/results/clientpositive/auto_join3.q.out +++ ql/src/test/results/clientpositive/auto_join3.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/auto_join4.q.out ql/src/test/results/clientpositive/auto_join4.q.out index 6c0fccdb6d..50f6f43fd4 100644 --- ql/src/test/results/clientpositive/auto_join4.q.out +++ ql/src/test/results/clientpositive/auto_join4.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join5.q.out ql/src/test/results/clientpositive/auto_join5.q.out index 410306175a..6077e1776b 100644 --- ql/src/test/results/clientpositive/auto_join5.q.out +++ ql/src/test/results/clientpositive/auto_join5.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join6.q.out ql/src/test/results/clientpositive/auto_join6.q.out index 53caf7d25e..48f8596a9a 100644 --- ql/src/test/results/clientpositive/auto_join6.q.out +++ ql/src/test/results/clientpositive/auto_join6.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join7.q.out ql/src/test/results/clientpositive/auto_join7.q.out index a657c301ee..3957b7e199 100644 --- ql/src/test/results/clientpositive/auto_join7.q.out +++ ql/src/test/results/clientpositive/auto_join7.q.out @@ -138,7 +138,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join8.q.out ql/src/test/results/clientpositive/auto_join8.q.out index 2689578135..2eb0d6d5d3 100644 --- ql/src/test/results/clientpositive/auto_join8.q.out +++ ql/src/test/results/clientpositive/auto_join8.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join9.q.out ql/src/test/results/clientpositive/auto_join9.q.out index d7d7d181f0..e5022a3d29 100644 --- ql/src/test/results/clientpositive/auto_join9.q.out +++ ql/src/test/results/clientpositive/auto_join9.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12' diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out index 8c7658c447..486d7b3d35 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -139,7 +140,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -303,7 +305,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -316,7 +319,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -480,7 +484,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -493,7 +498,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 diff --git ql/src/test/results/clientpositive/avro_decimal.q.out ql/src/test/results/clientpositive/avro_decimal.q.out index a9ce51b7d4..ee32c198a2 100644 --- ql/src/test/results/clientpositive/avro_decimal.q.out +++ ql/src/test/results/clientpositive/avro_decimal.q.out @@ -21,10 +21,12 @@ POSTHOOK: Output: default@dec PREHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value PREHOOK: type: QUERY PREHOOK: Input: default@dec +PREHOOK: Output: default@dec #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value POSTHOOK: type: QUERY POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec #### A masked pattern was here #### PREHOOK: query: DESC FORMATTED `dec` value PREHOOK: type: DESCTABLE @@ -35,7 +37,7 @@ POSTHOOK: Input: default@dec # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector value decimal(8,4) -12.25 234.79 0 10 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"value\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/avro_decimal_native.q.out ql/src/test/results/clientpositive/avro_decimal_native.q.out index 822882e662..dbe2a6647a 100644 --- ql/src/test/results/clientpositive/avro_decimal_native.q.out +++ ql/src/test/results/clientpositive/avro_decimal_native.q.out @@ -25,10 +25,12 @@ POSTHOOK: Output: default@dec PREHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value PREHOOK: type: QUERY PREHOOK: Input: default@dec +PREHOOK: Output: default@dec #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value POSTHOOK: type: QUERY POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec #### A masked pattern was here #### PREHOOK: query: DESC FORMATTED `dec` value PREHOOK: type: DESCTABLE @@ -39,7 +41,7 @@ POSTHOOK: Input: default@dec # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector value decimal(8,4) -12.25 234.79 0 10 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"value\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/binary_output_format.q.out ql/src/test/results/clientpositive/binary_output_format.q.out index ddb6adf4a0..ce85351c91 100644 --- ql/src/test/results/clientpositive/binary_output_format.q.out +++ ql/src/test/results/clientpositive/binary_output_format.q.out @@ -208,7 +208,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/bucket1.q.out ql/src/test/results/clientpositive/bucket1.q.out index 1d204731cd..c59a755428 100644 --- ql/src/test/results/clientpositive/bucket1.q.out +++ ql/src/test/results/clientpositive/bucket1.q.out @@ -160,7 +160,8 @@ STAGE PLANS: name: default.bucket1_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket1_1 diff --git ql/src/test/results/clientpositive/bucket2.q.out ql/src/test/results/clientpositive/bucket2.q.out index 48ccafb746..4bee31b05c 100644 --- ql/src/test/results/clientpositive/bucket2.q.out +++ ql/src/test/results/clientpositive/bucket2.q.out @@ -160,7 +160,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/bucket3.q.out ql/src/test/results/clientpositive/bucket3.q.out index b1173e7b35..49f12f7fe9 100644 --- ql/src/test/results/clientpositive/bucket3.q.out +++ ql/src/test/results/clientpositive/bucket3.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') diff --git ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out index 70cd53c6e5..f210c5a4d2 100644 --- ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out @@ -390,7 +390,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -724,7 +725,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out index d0c3a1aca0..c6d607916f 100644 --- ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out @@ -374,7 +374,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -708,7 +709,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out index eed4a5a970..52c17ccd32 100644 --- ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out @@ -374,7 +374,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -708,7 +709,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/bucketmapjoin5.q.out ql/src/test/results/clientpositive/bucketmapjoin5.q.out index 4b989932ce..bb447e311f 100644 --- ql/src/test/results/clientpositive/bucketmapjoin5.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin5.q.out @@ -415,7 +415,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -956,7 +957,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out index 97cb1f10d3..5227aca2e4 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out @@ -304,7 +304,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out index 724df736ee..af53b9bb6e 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out @@ -367,7 +367,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out index 165f0dc1e5..2d6bd6f709 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT x.key, x.value from @@ -187,7 +188,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * from @@ -292,7 +294,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') @@ -357,7 +360,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') @@ -411,5 +415,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out index c5e03be100..7eb36bfe33 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT x.value, x.key from @@ -196,7 +197,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT x.key, x.value from diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out index 1d794c3d28..7efb7cef02 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work @@ -427,7 +428,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out index 1e70105b9b..6c10249dea 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work @@ -437,7 +438,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out index f0e77f00af..cd8e3601ca 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work @@ -434,7 +435,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/case_sensitivity.q.out ql/src/test/results/clientpositive/case_sensitivity.q.out index b3969ccf90..c2f2f8b39f 100644 --- ql/src/test/results/clientpositive/case_sensitivity.q.out +++ ql/src/test/results/clientpositive/case_sensitivity.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/cast1.q.out ql/src/test/results/clientpositive/cast1.q.out index 9feb14f1bb..a0cd142bb9 100644 --- ql/src/test/results/clientpositive/cast1.q.out +++ ql/src/test/results/clientpositive/cast1.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out index 88b5d8464f..4b1644c5d7 100644 --- ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out @@ -76,10 +76,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select a, c, min(b) from ( select state as a, locid as b, count(*) as c @@ -177,10 +179,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select year from loc_orc group by year PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out index afc4fd90f9..7fddda8d73 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out @@ -49,10 +49,12 @@ POSTHOOK: Output: default@tbl1 PREHOOK: query: analyze table tbl1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tbl1 +PREHOOK: Output: default@tbl1 #### A masked pattern was here #### POSTHOOK: query: analyze table tbl1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl1 +POSTHOOK: Output: default@tbl1 #### A masked pattern was here #### PREHOOK: query: analyze table tbl2 compute statistics PREHOOK: type: QUERY @@ -65,10 +67,12 @@ POSTHOOK: Output: default@tbl2 PREHOOK: query: analyze table tbl2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tbl2 +PREHOOK: Output: default@tbl2 #### A masked pattern was here #### POSTHOOK: query: analyze table tbl2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl2 +POSTHOOK: Output: default@tbl2 #### A masked pattern was here #### PREHOOK: query: explain select count(*) from ( diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out index b29628038c..d3747afd9a 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* diff --git ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out index d4d70bc35e..e5d4421ba7 100644 --- ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 @@ -187,7 +188,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out index c09764c156..177eefd4ea 100644 --- ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/colstats_all_nulls.q.out ql/src/test/results/clientpositive/colstats_all_nulls.q.out index 9efab75d72..adc06666bd 100644 --- ql/src/test/results/clientpositive/colstats_all_nulls.q.out +++ ql/src/test/results/clientpositive/colstats_all_nulls.q.out @@ -30,10 +30,12 @@ POSTHOOK: Lineage: all_nulls.c SIMPLE [] PREHOOK: query: analyze table all_nulls compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@all_nulls +PREHOOK: Output: default@all_nulls #### A masked pattern was here #### POSTHOOK: query: analyze table all_nulls compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@all_nulls +POSTHOOK: Output: default@all_nulls #### A masked pattern was here #### PREHOOK: query: describe formatted all_nulls a PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out index 4e430b3f0b..49754787b8 100644 --- ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out +++ ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out @@ -56,10 +56,12 @@ Storage Desc Params: PREHOOK: query: analyze table calendar compute statistics for columns year PREHOOK: type: QUERY PREHOOK: Input: default@calendar +PREHOOK: Output: default@calendar #### A masked pattern was here #### POSTHOOK: query: analyze table calendar compute statistics for columns year POSTHOOK: type: QUERY POSTHOOK: Input: default@calendar +POSTHOOK: Output: default@calendar #### A masked pattern was here #### PREHOOK: query: desc formatted calendar PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out index c48042227e..37f855df5b 100644 --- ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out +++ ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out @@ -27,8 +27,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-2, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-3 Stage-3 depends on stages: Stage-1 STAGE PLANS: @@ -89,10 +88,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-4 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string diff --git ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out index 018e18fa9b..feac97e8eb 100644 --- ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out +++ ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out @@ -60,7 +60,8 @@ STAGE PLANS: name: default.columnarserde_create_shortcut Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE columnarserde_create_shortcut SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/columnstats_infinity.q.out ql/src/test/results/clientpositive/columnstats_infinity.q.out index 23ca4869a5..a1ec9b205a 100644 --- ql/src/test/results/clientpositive/columnstats_infinity.q.out +++ ql/src/test/results/clientpositive/columnstats_infinity.q.out @@ -118,10 +118,12 @@ Storage Desc Params: PREHOOK: query: analyze table table_change_numeric_group_string_group_floating_string_group compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@table_change_numeric_group_string_group_floating_string_group +PREHOOK: Output: default@table_change_numeric_group_string_group_floating_string_group #### A masked pattern was here #### POSTHOOK: query: analyze table table_change_numeric_group_string_group_floating_string_group compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@table_change_numeric_group_string_group_floating_string_group +POSTHOOK: Output: default@table_change_numeric_group_string_group_floating_string_group #### A masked pattern was here #### PREHOOK: query: desc formatted table_change_numeric_group_string_group_floating_string_group PREHOOK: type: DESCTABLE @@ -276,7 +278,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"c1\":\"true\",\"c10\":\"true\",\"c12\":\"true\",\"c13\":\"true\",\"c15\":\"true\",\"c3\":\"true\",\"c4\":\"true\",\"c6\":\"true\",\"c7\":\"true\",\"c9\":\"true\",\"insert_num\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"c1\":\"true\",\"c10\":\"true\",\"c11\":\"true\",\"c12\":\"true\",\"c13\":\"true\",\"c14\":\"true\",\"c15\":\"true\",\"c2\":\"true\",\"c3\":\"true\",\"c4\":\"true\",\"c5\":\"true\",\"c6\":\"true\",\"c7\":\"true\",\"c8\":\"true\",\"c9\":\"true\",\"insert_num\":\"true\"}} numFiles 1 numRows 5 rawDataSize 1250 diff --git ql/src/test/results/clientpositive/columnstats_partlvl.q.out ql/src/test/results/clientpositive/columnstats_partlvl.q.out index 1e1c55d4bf..b6075774f4 100644 --- ql/src/test/results/clientpositive/columnstats_partlvl.q.out +++ ql/src/test/results/clientpositive/columnstats_partlvl.q.out @@ -83,7 +83,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -106,7 +107,8 @@ STAGE PLANS: TableScan alias: employee_part Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.employee_part/ + GatherStats: true Select Operator expressions: employeeid (type: int) outputColumnNames: employeeid @@ -215,7 +217,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.employee_part/ Column Stats Desc: Columns: employeeID Column Types: int @@ -226,11 +230,15 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) co PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### PREHOOK: query: explain analyze table Employee_Part partition (employeeSalary=4000.0) compute statistics for columns employeeID @@ -285,7 +293,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -308,7 +317,8 @@ STAGE PLANS: TableScan alias: employee_part Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.employee_part/ + GatherStats: true Select Operator expressions: employeeid (type: int) outputColumnNames: employeeid @@ -417,7 +427,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.employee_part/ Column Stats Desc: Columns: employeeID Column Types: int @@ -428,11 +440,15 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary=4000.0) co PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary=4000.0) compute statistics for columns employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: explain analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns @@ -450,22 +466,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string) outputColumnNames: employeeid, employeename - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') keys: 2000.0 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 2000.0 (type: double) sort order: + Map-reduce partition columns: 2000.0 (type: double) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct), _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -473,21 +489,22 @@ STAGE PLANS: keys: 2000.0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: struct), _col2 (type: struct), 2000.0 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -497,11 +514,15 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) co PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary=2000.0) employeeID PREHOOK: type: DESCTABLE @@ -537,22 +558,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string), employeesalary (type: double) outputColumnNames: employeeid, employeename, employeesalary - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') keys: employeesalary (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct), _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -560,21 +581,22 @@ STAGE PLANS: keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -585,12 +607,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition(employeeSalary=2000.0) employeeID PREHOOK: type: DESCTABLE @@ -626,11 +654,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string) outputColumnNames: employeeid, employeename - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') mode: hash @@ -655,7 +683,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -666,12 +695,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part employeeID PREHOOK: type: DESCTABLE @@ -699,11 +734,16 @@ PREHOOK: query: analyze table default.Employee_Part partition (employeeSalary=20 PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### +Cannot get table employee_part POSTHOOK: query: analyze table default.Employee_Part partition (employeeSalary=2000.0) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted default.Employee_Part partition (employeeSalary=2000.0) employeeID PREHOOK: type: DESCTABLE @@ -720,12 +760,19 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### +Cannot get table employee_part POSTHOOK: query: analyze table default.Employee_Part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: use default PREHOOK: type: SWITCHDATABASE diff --git ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out index 0739b497f6..477b89e021 100644 --- ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out +++ ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out @@ -119,7 +119,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeName, employeeID Column Types: string, int @@ -129,11 +130,15 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary='4000.0', PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary='4000.0', country) compute statistics for columns employeeName, employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='4000.0', country='USA') employeeName PREHOOK: type: DESCTABLE @@ -197,7 +202,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -208,12 +214,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary='2000.0') compute statistics for columns employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='2000.0', country='USA') employeeID PREHOOK: type: DESCTABLE @@ -249,22 +261,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: employeeid (type: int), employeesalary (type: double), country (type: string) outputColumnNames: employeeid, employeesalary, country - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll') keys: employeesalary (type: double), country (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -272,21 +284,22 @@ STAGE PLANS: keys: KEY._col0 (type: double), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: struct), _col0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -301,6 +314,13 @@ PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA PREHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary) compute statistics for columns employeeID POSTHOOK: type: QUERY @@ -311,6 +331,13 @@ POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA POSTHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='3000.0', country='UK') employeeID PREHOOK: type: DESCTABLE @@ -337,22 +364,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 2 Data size: 466 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string), employeesalary (type: double), country (type: string) outputColumnNames: employeeid, employeename, employeesalary, country - Statistics: Num rows: 2 Data size: 466 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') keys: employeesalary (type: double), country (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 2 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Operator Tree: Group By Operator @@ -360,21 +387,22 @@ STAGE PLANS: keys: KEY._col0 (type: double), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -389,6 +417,13 @@ PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA PREHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary,country) compute statistics for columns POSTHOOK: type: QUERY @@ -399,6 +434,13 @@ POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA POSTHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='3500.0', country='UK') employeeName PREHOOK: type: DESCTABLE @@ -466,6 +508,11 @@ PREHOOK: Input: default@employee@employeesalary=2000.0/country=UK PREHOOK: Input: default@employee@employeesalary=2000.0/country=USA PREHOOK: Input: default@employee@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee +PREHOOK: Output: default@employee@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=3500.0/country=UK #### A masked pattern was here #### POSTHOOK: query: analyze table Employee partition (employeeSalary,country) compute statistics for columns POSTHOOK: type: QUERY @@ -474,6 +521,11 @@ POSTHOOK: Input: default@employee@employeesalary=2000.0/country=UK POSTHOOK: Input: default@employee@employeesalary=2000.0/country=USA POSTHOOK: Input: default@employee@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=3500.0/country=UK #### A masked pattern was here #### PREHOOK: query: describe formatted Employee partition (employeeSalary='3500.0', country='UK') employeeName PREHOOK: type: DESCTABLE @@ -511,6 +563,13 @@ PREHOOK: Input: default@employee@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee@employeesalary=3000.0/country=USA PREHOOK: Input: default@employee@employeesalary=3500.0/country=UK PREHOOK: Input: default@employee@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee +PREHOOK: Output: default@employee@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=3000.0/country=USA +PREHOOK: Output: default@employee@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee partition (employeeSalary) compute statistics for columns POSTHOOK: type: QUERY @@ -521,6 +580,13 @@ POSTHOOK: Input: default@employee@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee@employeesalary=3000.0/country=USA POSTHOOK: Input: default@employee@employeesalary=3500.0/country=UK POSTHOOK: Input: default@employee@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=3000.0/country=USA +POSTHOOK: Output: default@employee@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee partition (employeeSalary='3000.0', country='USA') employeeName PREHOOK: type: DESCTABLE @@ -552,11 +618,15 @@ PREHOOK: query: analyze table Employee partition (employeeSalary='6000.0',countr PREHOOK: type: QUERY PREHOOK: Input: default@employee PREHOOK: Input: default@employee@employeesalary=6000.0/country=UK +PREHOOK: Output: default@employee +PREHOOK: Output: default@employee@employeesalary=6000.0/country=UK #### A masked pattern was here #### POSTHOOK: query: analyze table Employee partition (employeeSalary='6000.0',country='UK') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee POSTHOOK: Input: default@employee@employeesalary=6000.0/country=UK +POSTHOOK: Output: default@employee +POSTHOOK: Output: default@employee@employeesalary=6000.0/country=UK #### A masked pattern was here #### PREHOOK: query: describe formatted Employee partition (employeeSalary='6000.0', country='UK') employeeName PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/columnstats_quoting.q.out ql/src/test/results/clientpositive/columnstats_quoting.q.out index b17ce0e67a..683c1e274f 100644 --- ql/src/test/results/clientpositive/columnstats_quoting.q.out +++ ql/src/test/results/clientpositive/columnstats_quoting.q.out @@ -53,7 +53,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: user id, user name Column Types: bigint, string @@ -62,10 +63,12 @@ STAGE PLANS: PREHOOK: query: analyze table user_web_events compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@user_web_events +PREHOOK: Output: default@user_web_events #### A masked pattern was here #### POSTHOOK: query: analyze table user_web_events compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@user_web_events +POSTHOOK: Output: default@user_web_events #### A masked pattern was here #### PREHOOK: query: explain analyze table user_web_events compute statistics for columns `user id` PREHOOK: type: QUERY @@ -110,7 +113,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: user id Column Types: bigint @@ -119,8 +123,10 @@ STAGE PLANS: PREHOOK: query: analyze table user_web_events compute statistics for columns `user id` PREHOOK: type: QUERY PREHOOK: Input: default@user_web_events +PREHOOK: Output: default@user_web_events #### A masked pattern was here #### POSTHOOK: query: analyze table user_web_events compute statistics for columns `user id` POSTHOOK: type: QUERY POSTHOOK: Input: default@user_web_events +POSTHOOK: Output: default@user_web_events #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/columnstats_tbllvl.q.out ql/src/test/results/clientpositive/columnstats_tbllvl.q.out index 25892d8fa6..1f62a64d1a 100644 --- ql/src/test/results/clientpositive/columnstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/columnstats_tbllvl.q.out @@ -83,7 +83,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -106,7 +107,8 @@ STAGE PLANS: TableScan alias: uservisits_web_text_none Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.uservisits_web_text_none/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite @@ -205,7 +207,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.uservisits_web_text_none/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -215,10 +219,12 @@ STAGE PLANS: PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: explain analyze table default.UserVisits_web_text_none compute statistics for columns @@ -236,11 +242,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_web_text_none - Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), desturl (type: string), visitdate (type: string), adrevenue (type: float), useragent (type: string), ccode (type: string), lcode (type: string), skeyword (type: string), avgtimeonsite (type: int) outputColumnNames: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite - Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(desturl, 'hll'), compute_stats(visitdate, 'hll'), compute_stats(adrevenue, 'hll'), compute_stats(useragent, 'hll'), compute_stats(ccode, 'hll'), compute_stats(lcode, 'hll'), compute_stats(skeyword, 'hll'), compute_stats(avgtimeonsite, 'hll') mode: hash @@ -265,7 +271,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite Column Types: string, string, string, float, string, string, string, string, int @@ -274,10 +281,12 @@ STAGE PLANS: PREHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: describe formatted UserVisits_web_text_none destURL PREHOOK: type: DESCTABLE @@ -288,7 +297,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector destURL string 0 55 48.945454545454545 96 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted UserVisits_web_text_none adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -298,7 +307,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector adRevenue float 13.099044799804688 492.98870849609375 0 55 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted UserVisits_web_text_none avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -308,7 +317,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector avgTimeOnSite int 1 9 0 9 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: CREATE TABLE empty_tab( a int, b double, @@ -374,7 +383,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d, e Column Types: int, double, string, boolean, binary @@ -383,10 +393,12 @@ STAGE PLANS: PREHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e PREHOOK: type: QUERY PREHOOK: Input: default@empty_tab +PREHOOK: Output: default@empty_tab #### A masked pattern was here #### POSTHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e POSTHOOK: type: QUERY POSTHOOK: Input: default@empty_tab +POSTHOOK: Output: default@empty_tab #### A masked pattern was here #### PREHOOK: query: create database if not exists dummydb PREHOOK: type: CREATEDATABASE @@ -403,10 +415,13 @@ POSTHOOK: Input: database:dummydb PREHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns destURL PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### +Cannot get table uservisits_web_text_none POSTHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns destURL POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: describe formatted default.UserVisits_web_text_none destURL PREHOOK: type: DESCTABLE @@ -417,7 +432,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector destURL string 0 55 48.945454545454545 96 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: CREATE TABLE UserVisits_in_dummy_db ( sourceIP string, destURL string, @@ -505,7 +520,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -528,7 +544,8 @@ STAGE PLANS: TableScan alias: uservisits_in_dummy_db Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: dummydb.uservisits_in_dummy_db/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite @@ -627,7 +644,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: dummydb.uservisits_in_dummy_db/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -637,10 +656,13 @@ STAGE PLANS: PREHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: dummydb@uservisits_in_dummy_db +PREHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### +Cannot get table uservisits_in_dummy_db POSTHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: dummydb@uservisits_in_dummy_db +POSTHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### PREHOOK: query: explain analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns @@ -658,11 +680,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_in_dummy_db - Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), desturl (type: string), visitdate (type: string), adrevenue (type: float), useragent (type: string), ccode (type: string), lcode (type: string), skeyword (type: string), avgtimeonsite (type: int) outputColumnNames: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite - Statistics: Num rows: 9 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(desturl, 'hll'), compute_stats(visitdate, 'hll'), compute_stats(adrevenue, 'hll'), compute_stats(useragent, 'hll'), compute_stats(ccode, 'hll'), compute_stats(lcode, 'hll'), compute_stats(skeyword, 'hll'), compute_stats(avgtimeonsite, 'hll') mode: hash @@ -687,7 +709,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite Column Types: string, string, string, float, string, string, string, string, int @@ -696,10 +719,13 @@ STAGE PLANS: PREHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: dummydb@uservisits_in_dummy_db +PREHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### +Cannot get table uservisits_in_dummy_db POSTHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: dummydb@uservisits_in_dummy_db +POSTHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### PREHOOK: query: describe formatted dummydb.UserVisits_in_dummy_db destURL PREHOOK: type: DESCTABLE @@ -710,7 +736,7 @@ POSTHOOK: Input: dummydb@uservisits_in_dummy_db # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector destURL string 0 55 48.945454545454545 96 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted dummydb.UserVisits_in_dummy_db adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: dummydb@uservisits_in_dummy_db @@ -720,7 +746,7 @@ POSTHOOK: Input: dummydb@uservisits_in_dummy_db # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector adRevenue float 13.099044799804688 492.98870849609375 0 55 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted dummydb.UserVisits_in_dummy_db avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: dummydb@uservisits_in_dummy_db @@ -730,7 +756,7 @@ POSTHOOK: Input: dummydb@uservisits_in_dummy_db # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector avgTimeOnSite int 1 9 0 9 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: drop table dummydb.UserVisits_in_dummy_db PREHOOK: type: DROPTABLE PREHOOK: Input: dummydb@uservisits_in_dummy_db diff --git ql/src/test/results/clientpositive/compustat_avro.q.out ql/src/test/results/clientpositive/compustat_avro.q.out index db20bef04e..aa7473253d 100644 --- ql/src/test/results/clientpositive/compustat_avro.q.out +++ ql/src/test/results/clientpositive/compustat_avro.q.out @@ -37,10 +37,12 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"tru PREHOOK: query: analyze table testAvro compute statistics for columns col1,col3 PREHOOK: type: QUERY PREHOOK: Input: default@testavro +PREHOOK: Output: default@testavro #### A masked pattern was here #### POSTHOOK: query: analyze table testAvro compute statistics for columns col1,col3 POSTHOOK: type: QUERY POSTHOOK: Input: default@testavro +POSTHOOK: Output: default@testavro #### A masked pattern was here #### PREHOOK: query: describe formatted testAvro col1 PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/compute_stats_date.q.out ql/src/test/results/clientpositive/compute_stats_date.q.out index e738c25fc9..dd26cfd3d2 100644 --- ql/src/test/results/clientpositive/compute_stats_date.q.out +++ ql/src/test/results/clientpositive/compute_stats_date.q.out @@ -89,7 +89,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: fl_date Column Types: date @@ -98,10 +99,12 @@ STAGE PLANS: PREHOOK: query: analyze table tab_date compute statistics for columns fl_date PREHOOK: type: QUERY PREHOOK: Input: default@tab_date +PREHOOK: Output: default@tab_date #### A masked pattern was here #### POSTHOOK: query: analyze table tab_date compute statistics for columns fl_date POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_date +POSTHOOK: Output: default@tab_date #### A masked pattern was here #### PREHOOK: query: describe formatted tab_date fl_date PREHOOK: type: DESCTABLE @@ -112,7 +115,7 @@ POSTHOOK: Input: default@tab_date # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector fl_date date 2000-11-20 2010-10-29 0 19 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"fl_date\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"fl_date\":\"true\"}} PREHOOK: query: alter table tab_date update statistics for column fl_date set ('numDVs'='19', 'highValue'='2015-01-01', 'lowValue'='0') PREHOOK: type: ALTERTABLE_UPDATETABLESTATS POSTHOOK: query: alter table tab_date update statistics for column fl_date set ('numDVs'='19', 'highValue'='2015-01-01', 'lowValue'='0') @@ -126,4 +129,4 @@ POSTHOOK: Input: default@tab_date # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector fl_date date 1970-01-01 2015-01-01 0 19 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"fl_date\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"fl_date\":\"true\"}} diff --git ql/src/test/results/clientpositive/constGby.q.out ql/src/test/results/clientpositive/constGby.q.out index 7115be3611..c633624935 100644 --- ql/src/test/results/clientpositive/constGby.q.out +++ ql/src/test/results/clientpositive/constGby.q.out @@ -17,10 +17,12 @@ POSTHOOK: Output: default@t1 PREHOOK: query: analyze table t1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 #### A masked pattern was here #### POSTHOOK: query: analyze table t1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 #### A masked pattern was here #### PREHOOK: query: explain select count(1) from t1 group by 1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/constant_prop_2.q.out ql/src/test/results/clientpositive/constant_prop_2.q.out index 77a91222fb..7c452451bd 100644 --- ql/src/test/results/clientpositive/constant_prop_2.q.out +++ ql/src/test/results/clientpositive/constant_prop_2.q.out @@ -74,7 +74,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string diff --git ql/src/test/results/clientpositive/constant_prop_3.q.out ql/src/test/results/clientpositive/constant_prop_3.q.out index 8119ccf028..88f25ff717 100644 --- ql/src/test/results/clientpositive/constant_prop_3.q.out +++ ql/src/test/results/clientpositive/constant_prop_3.q.out @@ -51,10 +51,12 @@ POSTHOOK: Output: default@part_hive PREHOOK: query: analyze table part_hive compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@part_hive +PREHOOK: Output: default@part_hive #### A masked pattern was here #### POSTHOOK: query: analyze table part_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@part_hive +POSTHOOK: Output: default@part_hive #### A masked pattern was here #### PREHOOK: query: analyze table partsupp_hive compute statistics PREHOOK: type: QUERY @@ -67,10 +69,12 @@ POSTHOOK: Output: default@partsupp_hive PREHOOK: query: analyze table partsupp_hive compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@partsupp_hive +PREHOOK: Output: default@partsupp_hive #### A masked pattern was here #### POSTHOOK: query: analyze table partsupp_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partsupp_hive +POSTHOOK: Output: default@partsupp_hive #### A masked pattern was here #### PREHOOK: query: analyze table supplier_hive compute statistics PREHOOK: type: QUERY @@ -83,10 +87,12 @@ POSTHOOK: Output: default@supplier_hive PREHOOK: query: analyze table supplier_hive compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@supplier_hive +PREHOOK: Output: default@supplier_hive #### A masked pattern was here #### POSTHOOK: query: analyze table supplier_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@supplier_hive +POSTHOOK: Output: default@supplier_hive #### A masked pattern was here #### Warning: Shuffle Join JOIN[25][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select diff --git ql/src/test/results/clientpositive/constprog_dp.q.out ql/src/test/results/clientpositive/constprog_dp.q.out index 8cf301d6f7..86b6c326d2 100644 --- ql/src/test/results/clientpositive/constprog_dp.q.out +++ ql/src/test/results/clientpositive/constprog_dp.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/constprog_type.q.out ql/src/test/results/clientpositive/constprog_type.q.out index d145d37a3b..27ef1f482b 100644 --- ql/src/test/results/clientpositive/constprog_type.q.out +++ ql/src/test/results/clientpositive/constprog_type.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/correlated_join_keys.q.out ql/src/test/results/clientpositive/correlated_join_keys.q.out index 5464cd3e1a..90f5860027 100644 --- ql/src/test/results/clientpositive/correlated_join_keys.q.out +++ ql/src/test/results/clientpositive/correlated_join_keys.q.out @@ -61,10 +61,12 @@ POSTHOOK: Output: default@customer_address PREHOOK: query: analyze table customer_address compute statistics for columns ca_state, ca_zip PREHOOK: type: QUERY PREHOOK: Input: default@customer_address +PREHOOK: Output: default@customer_address #### A masked pattern was here #### POSTHOOK: query: analyze table customer_address compute statistics for columns ca_state, ca_zip POSTHOOK: type: QUERY POSTHOOK: Input: default@customer_address +POSTHOOK: Output: default@customer_address #### A masked pattern was here #### PREHOOK: query: explain select count(*) from customer_address a join customer_address b on (a.ca_zip = b.ca_zip and a.ca_state = b.ca_state) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/correlationoptimizer5.q.out ql/src/test/results/clientpositive/correlationoptimizer5.q.out index 00bdb4caa1..9037b106ab 100644 --- ql/src/test/results/clientpositive/correlationoptimizer5.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer5.q.out @@ -210,7 +210,8 @@ STAGE PLANS: name: default.dest_co1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -461,7 +462,8 @@ STAGE PLANS: name: default.dest_co2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_co2 SELECT b.key, d.val @@ -632,7 +634,8 @@ STAGE PLANS: name: default.dest_co3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-13 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/cp_sel.q.out ql/src/test/results/clientpositive/cp_sel.q.out index af2efeb0cf..910003778a 100644 --- ql/src/test/results/clientpositive/cp_sel.q.out +++ ql/src/test/results/clientpositive/cp_sel.q.out @@ -128,7 +128,8 @@ STAGE PLANS: name: default.testpartbucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table testpartbucket partition(ds,hr) select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/ctas.q.out ql/src/test/results/clientpositive/ctas.q.out index c1b0838060..46e7b8f102 100644 --- ql/src/test/results/clientpositive/ctas.q.out +++ ql/src/test/results/clientpositive/ctas.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.nzhang_CTAS1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -249,7 +250,8 @@ STAGE PLANS: name: default.nzhang_ctas2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas2 as select * from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -400,7 +402,8 @@ STAGE PLANS: name: default.nzhang_ctas3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -616,7 +619,8 @@ STAGE PLANS: name: default.nzhang_ctas4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -771,7 +775,8 @@ STAGE PLANS: name: default.nzhang_ctas5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/ctas_colname.q.out ql/src/test/results/clientpositive/ctas_colname.q.out index 8d61c9dfdd..6399932d7d 100644 --- ql/src/test/results/clientpositive/ctas_colname.q.out +++ ql/src/test/results/clientpositive/ctas_colname.q.out @@ -62,7 +62,8 @@ STAGE PLANS: name: default.summary Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT @@ -222,7 +223,8 @@ STAGE PLANS: name: default.x4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x4 as select *, rank() over(partition by key order by value) as rr from src1 PREHOOK: type: CREATETABLE_AS_SELECT @@ -413,7 +415,8 @@ STAGE PLANS: name: default.x5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x5 as select *, lead(key,1) over(partition by key order by value) as lead1 from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT @@ -554,7 +557,8 @@ STAGE PLANS: name: default.x6 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -735,7 +739,8 @@ STAGE PLANS: name: default.x7 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x7 as select * from (select *, count(value) from src group by key, value) a PREHOOK: type: CREATETABLE_AS_SELECT @@ -1169,7 +1174,8 @@ STAGE PLANS: name: default.x8 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x8 as select * from (select *, count(value) from src group by key, value having key < 9) a PREHOOK: type: CREATETABLE_AS_SELECT @@ -1306,7 +1312,8 @@ STAGE PLANS: name: default.x9 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x9 as select * from (select max(value),key from src group by key having key < 9 AND max(value) IS NOT NULL) a PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/ctas_uses_database_location.q.out ql/src/test/results/clientpositive/ctas_uses_database_location.q.out index ed6d2bce3d..990a8210e9 100644 --- ql/src/test/results/clientpositive/ctas_uses_database_location.q.out +++ ql/src/test/results/clientpositive/ctas_uses_database_location.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: db1.table_db1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/decimal_stats.q.out ql/src/test/results/clientpositive/decimal_stats.q.out index eacad98e72..5350036091 100644 --- ql/src/test/results/clientpositive/decimal_stats.q.out +++ ql/src/test/results/clientpositive/decimal_stats.q.out @@ -35,10 +35,12 @@ POSTHOOK: Lineage: decimal_1.v EXPRESSION [] PREHOOK: query: analyze table decimal_1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 #### A masked pattern was here #### POSTHOOK: query: analyze table decimal_1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 #### A masked pattern was here #### PREHOOK: query: desc formatted decimal_1 v PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/deleteAnalyze.q.out ql/src/test/results/clientpositive/deleteAnalyze.q.out index d3609f8b6f..3c5576c006 100644 --- ql/src/test/results/clientpositive/deleteAnalyze.q.out +++ ql/src/test/results/clientpositive/deleteAnalyze.q.out @@ -79,18 +79,22 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 +PREHOOK: Output: default@testdeci2 #### A masked pattern was here #### POSTHOOK: query: analyze table testdeci2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 +POSTHOOK: Output: default@testdeci2 #### A masked pattern was here #### PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 +PREHOOK: Output: default@testdeci2 #### A masked pattern was here #### POSTHOOK: query: analyze table testdeci2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 +POSTHOOK: Output: default@testdeci2 #### A masked pattern was here #### PREHOOK: query: explain select s.id, diff --git ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out index 8cbed9accf..33302cb118 100644 --- ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out @@ -99,7 +99,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -122,7 +123,8 @@ STAGE PLANS: TableScan alias: uservisits_web_text_none Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.uservisits_web_text_none/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite @@ -221,7 +223,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.uservisits_web_text_none/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -231,10 +235,12 @@ STAGE PLANS: PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE @@ -245,7 +251,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sourceIP string 0 55 12.763636363636364 13 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -255,7 +261,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector avgTimeOnSite int 1 9 0 9 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -265,7 +271,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector adRevenue float 13.099044799804688 492.98870849609375 0 55 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: CREATE TABLE empty_tab( a int, b double, @@ -341,7 +347,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d, e Column Types: int, double, string, boolean, binary @@ -350,10 +357,12 @@ STAGE PLANS: PREHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e PREHOOK: type: QUERY PREHOOK: Input: default@empty_tab +PREHOOK: Output: default@empty_tab #### A masked pattern was here #### POSTHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e POSTHOOK: type: QUERY POSTHOOK: Input: default@empty_tab +POSTHOOK: Output: default@empty_tab #### A masked pattern was here #### PREHOOK: query: desc formatted empty_tab a PREHOOK: type: DESCTABLE @@ -444,7 +453,7 @@ POSTHOOK: query: desc extended default.UserVisits_web_text_none sourceIP POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@uservisits_web_text_none sourceIP string from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -472,15 +481,17 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sourceIP string 0 55 12.763636363636364 13 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword PREHOOK: type: QUERY PREHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +PREHOOK: Output: test@uservisits_web_text_none POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword POSTHOOK: type: QUERY POSTHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +POSTHOOK: Output: test@uservisits_web_text_none PREHOOK: query: desc extended UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -488,7 +499,7 @@ POSTHOOK: query: desc extended UserVisits_web_text_none sKeyword POSTHOOK: type: DESCTABLE POSTHOOK: Input: test@uservisits_web_text_none sKeyword string from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"skeyword\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"skeyword\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -498,7 +509,7 @@ POSTHOOK: Input: test@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sKeyword string 0 54 7.872727272727273 19 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"skeyword\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"skeyword\":\"true\"}} PREHOOK: query: desc formatted test.UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -508,4 +519,4 @@ POSTHOOK: Input: test@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sKeyword string 0 54 7.872727272727273 19 HL from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"skeyword\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"skeyword\":\"true\"}} diff --git ql/src/test/results/clientpositive/distinct_stats.q.out ql/src/test/results/clientpositive/distinct_stats.q.out index 73b4addf57..bc2ab02ef5 100644 --- ql/src/test/results/clientpositive/distinct_stats.q.out +++ ql/src/test/results/clientpositive/distinct_stats.q.out @@ -19,10 +19,12 @@ POSTHOOK: Lineage: t1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, co PREHOOK: query: analyze table t1 compute statistics for columns a,b PREHOOK: type: QUERY PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 #### A masked pattern was here #### POSTHOOK: query: analyze table t1 compute statistics for columns a,b POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 #### A masked pattern was here #### PREHOOK: query: explain select count(distinct b) from t1 group by a diff --git ql/src/test/results/clientpositive/drop_table_with_stats.q.out ql/src/test/results/clientpositive/drop_table_with_stats.q.out index 52aa10a070..119f74a87e 100644 --- ql/src/test/results/clientpositive/drop_table_with_stats.q.out +++ ql/src/test/results/clientpositive/drop_table_with_stats.q.out @@ -30,10 +30,12 @@ PREHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb1@testtable #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb1@testtable POSTHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb1@testtable #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb1@testtable PREHOOK: query: CREATE TABLE IF NOT EXISTS TestTable1 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:tblstatsdb1 @@ -54,10 +56,12 @@ PREHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb1@testtable1 POSTHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb1@testtable1 PREHOOK: query: CREATE TABLE IF NOT EXISTS TESTTABLE2 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:tblstatsdb1 @@ -78,10 +82,12 @@ PREHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb1@testtable2 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb1@testtable2 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb1@testtable2 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb1@testtable2 PREHOOK: query: DROP TABLE tblstatsdb1.testtable PREHOOK: type: DROPTABLE PREHOOK: Input: tblstatsdb1@testtable @@ -146,10 +152,12 @@ PREHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb2@testtable #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb2@testtable POSTHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb2@testtable #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb2@testtable PREHOOK: query: CREATE TABLE IF NOT EXISTS TestTable1 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: TBLSTATSDB2@TestTable1 @@ -170,10 +178,12 @@ PREHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb2@testtable1 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb2@testtable1 POSTHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb2@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb2@testtable1 PREHOOK: query: CREATE TABLE IF NOT EXISTS TESTTABLE2 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: TBLSTATSDB2@TESTTABLE2 @@ -194,10 +204,12 @@ PREHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb2@testtable2 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb2@testtable2 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb2@testtable2 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb2@testtable2 PREHOOK: query: DROP TABLE TBLSTATSDB2.testtable PREHOOK: type: DROPTABLE PREHOOK: Input: tblstatsdb2@testtable diff --git ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 76d0b7b02f..8199235519 100644 --- ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -69,5 +69,6 @@ STAGE PLANS: name: default.non_acid Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/encrypted/encryption_insert_values.q.out ql/src/test/results/clientpositive/encrypted/encryption_insert_values.q.out index e8645883b7..abf8deeb17 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_insert_values.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_insert_values.q.out @@ -49,6 +49,12 @@ Database: default Retention: 0 #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table/.hive-staging Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"tmp_values_col1\":\"true\",\"tmp_values_col2\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 # Storage Information SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out index 5ecf20d18f..d421a2b376 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out @@ -48,10 +48,12 @@ src PREHOOK: query: ANALYZE TABLE encrypted_table COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table #### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging POSTHOOK: query: ANALYZE TABLE encrypted_table COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table #### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging PREHOOK: query: DESCRIBE FORMATTED encrypted_table key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out index 5fd06afe90..90d42fb468 100644 --- ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out +++ ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out @@ -1,6 +1,18 @@ -PREHOOK: query: explain analyze table src compute statistics for columns +PREHOOK: query: create table t as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain analyze table t compute statistics for columns PREHOOK: type: QUERY -POSTHOOK: query: explain analyze table src compute statistics for columns +POSTHOOK: query: explain analyze table t compute statistics for columns POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage @@ -11,7 +23,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: src + alias: t Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -41,17 +53,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string - Table: default.src + Table: default.t -PREHOOK: query: analyze table src compute statistics for columns +PREHOOK: query: analyze table t compute statistics for columns PREHOOK: type: QUERY -PREHOOK: Input: default@src +PREHOOK: Input: default@t +PREHOOK: Output: default@t #### A masked pattern was here #### -POSTHOOK: query: analyze table src compute statistics for columns +POSTHOOK: query: analyze table t compute statistics for columns POSTHOOK: type: QUERY -POSTHOOK: Input: default@src +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/explain_ddl.q.out ql/src/test/results/clientpositive/explain_ddl.q.out index e108e2207c..f0e54c5153 100644 --- ql/src/test/results/clientpositive/explain_ddl.q.out +++ ql/src/test/results/clientpositive/explain_ddl.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.M1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -194,7 +195,8 @@ STAGE PLANS: name: default.M1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -288,7 +290,8 @@ STAGE PLANS: name: default.M1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -380,7 +383,8 @@ STAGE PLANS: name: default.V1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -540,7 +544,8 @@ STAGE PLANS: name: default.m1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out index b12d3a1f20..5f427375da 100644 --- ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out +++ ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out @@ -33,6 +33,10 @@ PREHOOK: Input: default@date_dim PREHOOK: Input: default@date_dim@d_date_sk=2416945 PREHOOK: Input: default@date_dim@d_date_sk=2416946 PREHOOK: Input: default@date_dim@d_date_sk=2416947 +PREHOOK: Output: default@date_dim +PREHOOK: Output: default@date_dim@d_date_sk=2416945 +PREHOOK: Output: default@date_dim@d_date_sk=2416946 +PREHOOK: Output: default@date_dim@d_date_sk=2416947 #### A masked pattern was here #### POSTHOOK: query: analyze table date_dim partition(d_date_sk) compute statistics for columns POSTHOOK: type: QUERY @@ -40,6 +44,10 @@ POSTHOOK: Input: default@date_dim POSTHOOK: Input: default@date_dim@d_date_sk=2416945 POSTHOOK: Input: default@date_dim@d_date_sk=2416946 POSTHOOK: Input: default@date_dim@d_date_sk=2416947 +POSTHOOK: Output: default@date_dim +POSTHOOK: Output: default@date_dim@d_date_sk=2416945 +POSTHOOK: Output: default@date_dim@d_date_sk=2416946 +POSTHOOK: Output: default@date_dim@d_date_sk=2416947 #### A masked pattern was here #### PREHOOK: query: explain select count(*) from date_dim where d_date > date "1900-01-02" and d_date_sk= 2416945 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out index 219e1285d2..e748770088 100644 --- ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out +++ ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out @@ -59,21 +59,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2001') state PREHOOK: type: DESCTABLE @@ -347,41 +355,57 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### PREHOOK: query: explain extended select state from loc_orc_2d PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out index b8bb210ae5..521f93ee5a 100644 --- ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out +++ ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out @@ -67,21 +67,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2002') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2002 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2002 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2002') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2002 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2002 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2001') state PREHOOK: type: DESCTABLE @@ -511,21 +519,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2003') compute statistics for columns state PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2003 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2003 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2003') compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2003 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2003 #### A masked pattern was here #### PREHOOK: query: explain extended select state from loc_orc_1d PREHOOK: type: QUERY @@ -991,21 +1007,29 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2002') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2002 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2002 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2002') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2002 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2002 #### A masked pattern was here #### PREHOOK: query: explain extended select state from loc_orc_2d PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/fm-sketch.q.out ql/src/test/results/clientpositive/fm-sketch.q.out index 7a4d395fb4..1aa81e43ca 100644 --- ql/src/test/results/clientpositive/fm-sketch.q.out +++ ql/src/test/results/clientpositive/fm-sketch.q.out @@ -58,7 +58,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -67,10 +68,12 @@ STAGE PLANS: PREHOOK: query: analyze table n compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@n +PREHOOK: Output: default@n #### A masked pattern was here #### POSTHOOK: query: analyze table n compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@n +POSTHOOK: Output: default@n #### A masked pattern was here #### PREHOOK: query: desc formatted n key PREHOOK: type: DESCTABLE @@ -142,7 +145,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -151,10 +155,12 @@ STAGE PLANS: PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -194,10 +200,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -237,10 +245,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -306,10 +316,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(values__tmp__table__5)values__tmp__table__ PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/groupby1.q.out ql/src/test/results/clientpositive/groupby1.q.out index 46e09dd256..d05feec784 100644 --- ql/src/test/results/clientpositive/groupby1.q.out +++ ql/src/test/results/clientpositive/groupby1.q.out @@ -90,7 +90,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby10.q.out ql/src/test/results/clientpositive/groupby10.q.out index 66832b02fc..b4656838e9 100644 --- ql/src/test/results/clientpositive/groupby10.q.out +++ ql/src/test/results/clientpositive/groupby10.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -197,7 +198,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -376,7 +378,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -442,7 +445,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -604,7 +608,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -617,7 +622,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, sum(distinct substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key diff --git ql/src/test/results/clientpositive/groupby11.q.out ql/src/test/results/clientpositive/groupby11.q.out index 1d0e86ab7d..2ab4c39c89 100644 --- ql/src/test/results/clientpositive/groupby11.q.out +++ ql/src/test/results/clientpositive/groupby11.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -189,7 +190,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 partition(ds='111') diff --git ql/src/test/results/clientpositive/groupby12.q.out ql/src/test/results/clientpositive/groupby12.q.out index 921fc92b3c..2f633f01a7 100644 --- ql/src/test/results/clientpositive/groupby12.q.out +++ ql/src/test/results/clientpositive/groupby12.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT COUNT(src.key), COUNT(DISTINCT value) GROUP BY src.key diff --git ql/src/test/results/clientpositive/groupby1_limit.q.out ql/src/test/results/clientpositive/groupby1_limit.q.out index 78a49ebf6c..e5fa6d1451 100644 --- ql/src/test/results/clientpositive/groupby1_limit.q.out +++ ql/src/test/results/clientpositive/groupby1_limit.q.out @@ -100,7 +100,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key ORDER BY src.key LIMIT 5 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_map.q.out ql/src/test/results/clientpositive/groupby1_map.q.out index cc985a5def..337c2e0fe1 100644 --- ql/src/test/results/clientpositive/groupby1_map.q.out +++ ql/src/test/results/clientpositive/groupby1_map.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_map_nomap.q.out ql/src/test/results/clientpositive/groupby1_map_nomap.q.out index cc985a5def..337c2e0fe1 100644 --- ql/src/test/results/clientpositive/groupby1_map_nomap.q.out +++ ql/src/test/results/clientpositive/groupby1_map_nomap.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_map_skew.q.out ql/src/test/results/clientpositive/groupby1_map_skew.q.out index 116744a29e..a140a0211f 100644 --- ql/src/test/results/clientpositive/groupby1_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby1_map_skew.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_noskew.q.out ql/src/test/results/clientpositive/groupby1_noskew.q.out index 98c0d3c28e..702ea3e701 100644 --- ql/src/test/results/clientpositive/groupby1_noskew.q.out +++ ql/src/test/results/clientpositive/groupby1_noskew.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby2_map.q.out ql/src/test/results/clientpositive/groupby2_map.q.out index 0dcd8109f1..427590d2ba 100644 --- ql/src/test/results/clientpositive/groupby2_map.q.out +++ ql/src/test/results/clientpositive/groupby2_map.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out index 64477dbfd2..0ab198501d 100644 --- ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) @@ -175,7 +176,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.key,1,1)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_map_skew.q.out ql/src/test/results/clientpositive/groupby2_map_skew.q.out index 813ae5cb26..bb5a09f4b3 100644 --- ql/src/test/results/clientpositive/groupby2_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby2_map_skew.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_noskew.q.out ql/src/test/results/clientpositive/groupby2_noskew.q.out index 5192db3966..782a1a2a75 100644 --- ql/src/test/results/clientpositive/groupby2_noskew.q.out +++ ql/src/test/results/clientpositive/groupby2_noskew.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out index 1c24213fba..9a070aca3f 100644 --- ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby3.q.out ql/src/test/results/clientpositive/groupby3.q.out index 2ebeae450b..2f2b533d8a 100644 --- ql/src/test/results/clientpositive/groupby3.q.out +++ ql/src/test/results/clientpositive/groupby3.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_map.q.out ql/src/test/results/clientpositive/groupby3_map.q.out index 07c122e2d9..1bbf23dd5b 100644 --- ql/src/test/results/clientpositive/groupby3_map.q.out +++ ql/src/test/results/clientpositive/groupby3_map.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out index a4501f7f7a..248a31746c 100644 --- ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_map_skew.q.out ql/src/test/results/clientpositive/groupby3_map_skew.q.out index e02bdeb904..8ba3a5865a 100644 --- ql/src/test/results/clientpositive/groupby3_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby3_map_skew.q.out @@ -112,7 +112,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_noskew.q.out ql/src/test/results/clientpositive/groupby3_noskew.q.out index 624fd2db54..5c520bbdd1 100644 --- ql/src/test/results/clientpositive/groupby3_noskew.q.out +++ ql/src/test/results/clientpositive/groupby3_noskew.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out index a1d403d6cf..c5d121b4f9 100644 --- ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby4.q.out ql/src/test/results/clientpositive/groupby4.q.out index 3f77e47bd1..e8e7b88412 100644 --- ql/src/test/results/clientpositive/groupby4.q.out +++ ql/src/test/results/clientpositive/groupby4.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby4_map.q.out ql/src/test/results/clientpositive/groupby4_map.q.out index 97915e76f8..647242aaa4 100644 --- ql/src/test/results/clientpositive/groupby4_map.q.out +++ ql/src/test/results/clientpositive/groupby4_map.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby4_map_skew.q.out ql/src/test/results/clientpositive/groupby4_map_skew.q.out index ae83f7ac65..0857f22bbc 100644 --- ql/src/test/results/clientpositive/groupby4_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby4_map_skew.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby4_noskew.q.out ql/src/test/results/clientpositive/groupby4_noskew.q.out index c7db0d7016..5827ef06ba 100644 --- ql/src/test/results/clientpositive/groupby4_noskew.q.out +++ ql/src/test/results/clientpositive/groupby4_noskew.q.out @@ -61,7 +61,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby5.q.out ql/src/test/results/clientpositive/groupby5.q.out index 9bf01ee51b..995d30e9a1 100644 --- ql/src/test/results/clientpositive/groupby5.q.out +++ ql/src/test/results/clientpositive/groupby5.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/groupby5_map.q.out ql/src/test/results/clientpositive/groupby5_map.q.out index 5fbd3d7dad..393c4e2d54 100644 --- ql/src/test/results/clientpositive/groupby5_map.q.out +++ ql/src/test/results/clientpositive/groupby5_map.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby5_map_skew.q.out ql/src/test/results/clientpositive/groupby5_map_skew.q.out index 60b010b501..67767f5153 100644 --- ql/src/test/results/clientpositive/groupby5_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby5_map_skew.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby5_noskew.q.out ql/src/test/results/clientpositive/groupby5_noskew.q.out index 612a0f6112..dfecdb5cef 100644 --- ql/src/test/results/clientpositive/groupby5_noskew.q.out +++ ql/src/test/results/clientpositive/groupby5_noskew.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/groupby6.q.out ql/src/test/results/clientpositive/groupby6.q.out index b79022405b..afc23ecf9f 100644 --- ql/src/test/results/clientpositive/groupby6.q.out +++ ql/src/test/results/clientpositive/groupby6.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby6_map.q.out ql/src/test/results/clientpositive/groupby6_map.q.out index 4ba3772298..c0c8841adc 100644 --- ql/src/test/results/clientpositive/groupby6_map.q.out +++ ql/src/test/results/clientpositive/groupby6_map.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby6_map_skew.q.out ql/src/test/results/clientpositive/groupby6_map_skew.q.out index 5141c0d9b3..5bb8695de6 100644 --- ql/src/test/results/clientpositive/groupby6_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby6_map_skew.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby6_noskew.q.out ql/src/test/results/clientpositive/groupby6_noskew.q.out index fd796c7b74..0cafbd1256 100644 --- ql/src/test/results/clientpositive/groupby6_noskew.q.out +++ ql/src/test/results/clientpositive/groupby6_noskew.q.out @@ -61,7 +61,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby7_map.q.out ql/src/test/results/clientpositive/groupby7_map.q.out index 0ef29cd29f..b7b2c8ce5e 100644 --- ql/src/test/results/clientpositive/groupby7_map.q.out +++ ql/src/test/results/clientpositive/groupby7_map.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -145,7 +146,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out index 7c3b033a62..6f88027bc4 100644 --- ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -112,7 +113,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_map_skew.q.out ql/src/test/results/clientpositive/groupby7_map_skew.q.out index 4bfa52ed89..17eb283633 100644 --- ql/src/test/results/clientpositive/groupby7_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby7_map_skew.q.out @@ -128,7 +128,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -195,7 +196,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_noskew.q.out ql/src/test/results/clientpositive/groupby7_noskew.q.out index 6178f58f7e..2303b3366b 100644 --- ql/src/test/results/clientpositive/groupby7_noskew.q.out +++ ql/src/test/results/clientpositive/groupby7_noskew.q.out @@ -90,7 +90,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -133,7 +134,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out index f38c428781..ff29aec5ef 100644 --- ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -164,7 +165,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key ORDER BY SRC.key limit 10 diff --git ql/src/test/results/clientpositive/groupby8.q.out ql/src/test/results/clientpositive/groupby8.q.out index 1856a9252b..826064cd53 100644 --- ql/src/test/results/clientpositive/groupby8.q.out +++ ql/src/test/results/clientpositive/groupby8.q.out @@ -115,7 +115,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -181,7 +182,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -936,7 +938,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -1002,7 +1005,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby8_map.q.out ql/src/test/results/clientpositive/groupby8_map.q.out index f683a8ba71..abe813c2b6 100644 --- ql/src/test/results/clientpositive/groupby8_map.q.out +++ ql/src/test/results/clientpositive/groupby8_map.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -111,7 +112,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby8_map_skew.q.out ql/src/test/results/clientpositive/groupby8_map_skew.q.out index 5e60d3e924..a4de8b4e06 100644 --- ql/src/test/results/clientpositive/groupby8_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby8_map_skew.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -193,7 +194,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby8_noskew.q.out ql/src/test/results/clientpositive/groupby8_noskew.q.out index f683a8ba71..abe813c2b6 100644 --- ql/src/test/results/clientpositive/groupby8_noskew.q.out +++ ql/src/test/results/clientpositive/groupby8_noskew.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -111,7 +112,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby9.q.out ql/src/test/results/clientpositive/groupby9.q.out index 15ea1857c0..28032e324f 100644 --- ql/src/test/results/clientpositive/groupby9.q.out +++ ql/src/test/results/clientpositive/groupby9.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -143,7 +144,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -885,7 +887,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -927,7 +930,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -1669,7 +1673,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -1711,7 +1716,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -2454,7 +2460,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -2497,7 +2504,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -3239,7 +3247,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -3281,7 +3290,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby_complex_types.q.out ql/src/test/results/clientpositive/groupby_complex_types.q.out index b16a4ade68..107eea3a1d 100644 --- ql/src/test/results/clientpositive/groupby_complex_types.q.out +++ ql/src/test/results/clientpositive/groupby_complex_types.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -166,7 +167,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce @@ -205,7 +207,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) diff --git ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out index caa5395031..bfddf74adc 100644 --- ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -191,7 +192,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10 diff --git ql/src/test/results/clientpositive/groupby_cube1.q.out ql/src/test/results/clientpositive/groupby_cube1.q.out index 9acccf3b45..38243592b4 100644 --- ql/src/test/results/clientpositive/groupby_cube1.q.out +++ ql/src/test/results/clientpositive/groupby_cube1.q.out @@ -652,7 +652,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -720,7 +721,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with cube diff --git ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out index 1e13288c9a..0f77b3d1ca 100644 --- ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out +++ ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.t1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -142,5 +143,6 @@ STAGE PLANS: name: default.t2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/groupby_duplicate_key.q.out ql/src/test/results/clientpositive/groupby_duplicate_key.q.out index 6e19930927..d1d7f8d269 100644 --- ql/src/test/results/clientpositive/groupby_duplicate_key.q.out +++ ql/src/test/results/clientpositive/groupby_duplicate_key.q.out @@ -143,7 +143,8 @@ STAGE PLANS: name: default.dummy Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table dummy as select distinct key, "X" as dummy1, "X" as dummy2 from src tablesample (10 rows) diff --git ql/src/test/results/clientpositive/groupby_map_ppr.q.out ql/src/test/results/clientpositive/groupby_map_ppr.q.out index 0e5b394215..e0952c8cdf 100644 --- ql/src/test/results/clientpositive/groupby_map_ppr.q.out +++ ql/src/test/results/clientpositive/groupby_map_ppr.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out index dbcef22473..3c53a58455 100644 --- ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out index 5f02b04c38..6606edb9a1 100644 --- ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -143,7 +144,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table dest1 select key, count(distinct value) group by key diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out index 256784d3d7..ca1ebd7586 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out @@ -151,7 +151,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -164,7 +165,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -177,7 +179,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) @@ -389,7 +392,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -402,7 +406,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -415,7 +420,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-9 Map Reduce @@ -510,7 +516,8 @@ STAGE PLANS: name: default.dest_h2 Stage: Stage-11 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Move Operator @@ -523,7 +530,8 @@ STAGE PLANS: name: default.dest_h3 Stage: Stage-12 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out index 012b2114b4..daa047db85 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -117,7 +118,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out index e41d9ef021..cb621bc19d 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -129,7 +130,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -282,7 +284,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -295,7 +298,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -448,7 +452,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -461,7 +466,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -614,7 +620,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -627,7 +634,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 diff --git ql/src/test/results/clientpositive/groupby_position.q.out ql/src/test/results/clientpositive/groupby_position.q.out index 2ec9ecd614..3d42322192 100644 --- ql/src/test/results/clientpositive/groupby_position.q.out +++ ql/src/test/results/clientpositive/groupby_position.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -141,7 +142,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 @@ -289,7 +291,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -331,7 +334,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 diff --git ql/src/test/results/clientpositive/groupby_ppr.q.out ql/src/test/results/clientpositive/groupby_ppr.q.out index e645f5f598..8a18187eb7 100644 --- ql/src/test/results/clientpositive/groupby_ppr.q.out +++ ql/src/test/results/clientpositive/groupby_ppr.q.out @@ -221,7 +221,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out index f300095bd7..6e4501d022 100644 --- ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out @@ -221,7 +221,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src @@ -482,7 +483,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_rollup1.q.out ql/src/test/results/clientpositive/groupby_rollup1.q.out index e050d0af29..b7e93d9c25 100644 --- ql/src/test/results/clientpositive/groupby_rollup1.q.out +++ ql/src/test/results/clientpositive/groupby_rollup1.q.out @@ -497,7 +497,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -565,7 +566,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with rollup diff --git ql/src/test/results/clientpositive/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_1_23.q.out index 1f12c52f6a..80ec75aff5 100644 --- ql/src/test/results/clientpositive/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_1_23.q.out @@ -200,7 +200,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -585,7 +586,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -781,7 +783,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1155,7 +1158,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1537,7 +1541,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1923,7 +1928,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -2123,7 +2129,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -2335,7 +2342,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2595,7 +2603,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -3168,7 +3177,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -3591,7 +3601,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4099,7 +4110,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4293,7 +4305,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -4678,7 +4691,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5062,7 +5076,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5453,7 +5468,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5784,7 +5800,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -5797,7 +5814,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -5942,7 +5960,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -5955,7 +5974,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/groupby_sort_2.q.out ql/src/test/results/clientpositive/groupby_sort_2.q.out index bb6273e37d..9d4143660b 100644 --- ql/src/test/results/clientpositive/groupby_sort_2.q.out +++ ql/src/test/results/clientpositive/groupby_sort_2.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 SELECT val, count(1) FROM T1 GROUP BY val diff --git ql/src/test/results/clientpositive/groupby_sort_3.q.out ql/src/test/results/clientpositive/groupby_sort_3.q.out index 2dae25d085..67b53223a8 100644 --- ql/src/test/results/clientpositive/groupby_sort_3.q.out +++ ql/src/test/results/clientpositive/groupby_sort_3.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -237,7 +238,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/groupby_sort_4.q.out ql/src/test/results/clientpositive/groupby_sort_4.q.out index 70e8ac7d8d..0b269783a3 100644 --- ql/src/test/results/clientpositive/groupby_sort_4.q.out +++ ql/src/test/results/clientpositive/groupby_sort_4.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 SELECT key, count(1) FROM T1 GROUP BY key @@ -204,7 +205,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 SELECT key, val, count(1) FROM T1 GROUP BY key, val diff --git ql/src/test/results/clientpositive/groupby_sort_5.q.out ql/src/test/results/clientpositive/groupby_sort_5.q.out index db18928faa..41e1782724 100644 --- ql/src/test/results/clientpositive/groupby_sort_5.q.out +++ ql/src/test/results/clientpositive/groupby_sort_5.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -265,7 +266,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -436,7 +438,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 SELECT key, count(1) FROM T1 GROUP BY key diff --git ql/src/test/results/clientpositive/groupby_sort_6.q.out ql/src/test/results/clientpositive/groupby_sort_6.q.out index 60019e7f99..a66ec97642 100644 --- ql/src/test/results/clientpositive/groupby_sort_6.q.out +++ ql/src/test/results/clientpositive/groupby_sort_6.q.out @@ -132,7 +132,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -282,7 +283,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -470,7 +472,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 diff --git ql/src/test/results/clientpositive/groupby_sort_7.q.out ql/src/test/results/clientpositive/groupby_sort_7.q.out index 9d535e25b0..c9746c3126 100644 --- ql/src/test/results/clientpositive/groupby_sort_7.q.out +++ ql/src/test/results/clientpositive/groupby_sort_7.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out index fba8adbd49..3b3e2276ca 100644 --- ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out @@ -200,7 +200,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -654,7 +655,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -850,7 +852,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1224,7 +1227,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1606,7 +1610,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -2061,7 +2066,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -2330,7 +2336,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -2611,7 +2618,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2871,7 +2879,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -3513,7 +3522,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -3936,7 +3946,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4582,7 +4593,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4776,7 +4788,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5161,7 +5174,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5545,7 +5559,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5936,7 +5951,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -6292,7 +6308,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -6305,7 +6322,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -6475,7 +6493,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -6488,7 +6507,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/groupby_sort_test_1.q.out ql/src/test/results/clientpositive/groupby_sort_test_1.q.out index d06cd7cce0..1723e75467 100644 --- ql/src/test/results/clientpositive/groupby_sort_test_1.q.out +++ ql/src/test/results/clientpositive/groupby_sort_test_1.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/hll.q.out ql/src/test/results/clientpositive/hll.q.out index bb880b2d18..2e7fd280ee 100644 --- ql/src/test/results/clientpositive/hll.q.out +++ ql/src/test/results/clientpositive/hll.q.out @@ -58,7 +58,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -67,10 +68,12 @@ STAGE PLANS: PREHOOK: query: analyze table n compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@n +PREHOOK: Output: default@n #### A masked pattern was here #### POSTHOOK: query: analyze table n compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@n +POSTHOOK: Output: default@n #### A masked pattern was here #### PREHOOK: query: desc formatted n key PREHOOK: type: DESCTABLE @@ -142,7 +145,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -151,10 +155,12 @@ STAGE PLANS: PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -194,10 +200,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -237,10 +245,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -306,10 +316,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(values__tmp__table__5)values__tmp__table__ PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out index c4b9dc4c62..1fa9e94aa7 100644 --- ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out +++ ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.implicit_cast_during_insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table implicit_cast_during_insert partition (p1) select key, value, key key1 from (select * from src where key in (0,1)) q diff --git ql/src/test/results/clientpositive/index_auto_update.q.out ql/src/test/results/clientpositive/index_auto_update.q.out index e7bc0690ad..bc444cc9fb 100644 --- ql/src/test/results/clientpositive/index_auto_update.q.out +++ ql/src/test/results/clientpositive/index_auto_update.q.out @@ -135,14 +135,16 @@ STAGE PLANS: name: default.default__temp_temp_index__ Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Stage: Stage-5 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce diff --git ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out index d62d0b8d2f..4d3da307cd 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out @@ -498,7 +498,8 @@ STAGE PLANS: name: default.test_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out index a448ef312c..3de7003a86 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: SELECT key, value, count(1) FROM src GROUP BY ROLLUP (key, value) PREHOOK: type: QUERY @@ -1514,7 +1515,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT key, value, count(1) FROM src GROUP BY key, value WITH CUBE @@ -1692,7 +1694,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT key, value, count(1) FROM src GROUP BY key, value GROUPING SETS (key, value) diff --git ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out index 32edd73030..847c06150c 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out @@ -114,7 +114,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -295,7 +296,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT a.key, a.value FROM ( @@ -426,7 +428,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -581,7 +584,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT /*+ MAPJOIN(a) */ b.value, count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key diff --git ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out index 0c61fe0212..1ab6d356e2 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.test_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (ds = '2008-04-08', hr) SELECT key2, value, cast(hr as int) FROM diff --git ql/src/test/results/clientpositive/innerjoin.q.out ql/src/test/results/clientpositive/innerjoin.q.out index 99b3d856eb..741c9fbbd4 100644 --- ql/src/test/results/clientpositive/innerjoin.q.out +++ ql/src/test/results/clientpositive/innerjoin.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 INNER JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/input11.q.out ql/src/test/results/clientpositive/input11.q.out index bb22ee86df..356cef120a 100644 --- ql/src/test/results/clientpositive/input11.q.out +++ ql/src/test/results/clientpositive/input11.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input11_limit.q.out ql/src/test/results/clientpositive/input11_limit.q.out index 597554e02f..8fb7b630fc 100644 --- ql/src/test/results/clientpositive/input11_limit.q.out +++ ql/src/test/results/clientpositive/input11_limit.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 LIMIT 10 diff --git ql/src/test/results/clientpositive/input12.q.out ql/src/test/results/clientpositive/input12.q.out index 3bb765cf78..a6af3e675d 100644 --- ql/src/test/results/clientpositive/input12.q.out +++ ql/src/test/results/clientpositive/input12.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -183,7 +184,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-11 Map Reduce @@ -238,7 +240,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-16 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-17 Map Reduce diff --git ql/src/test/results/clientpositive/input13.q.out ql/src/test/results/clientpositive/input13.q.out index 91ff23429b..c414ff2ab5 100644 --- ql/src/test/results/clientpositive/input13.q.out +++ ql/src/test/results/clientpositive/input13.q.out @@ -153,7 +153,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -205,7 +206,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-11 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-12 Map Reduce @@ -260,7 +262,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-17 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-18 Map Reduce diff --git ql/src/test/results/clientpositive/input14.q.out ql/src/test/results/clientpositive/input14.q.out index af04a9896d..d8455267eb 100644 --- ql/src/test/results/clientpositive/input14.q.out +++ ql/src/test/results/clientpositive/input14.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input14_limit.q.out ql/src/test/results/clientpositive/input14_limit.q.out index 9870ad53b4..f62ad051c7 100644 --- ql/src/test/results/clientpositive/input14_limit.q.out +++ ql/src/test/results/clientpositive/input14_limit.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input17.q.out ql/src/test/results/clientpositive/input17.q.out index 057a92d163..316c9c6c6b 100644 --- ql/src/test/results/clientpositive/input17.q.out +++ ql/src/test/results/clientpositive/input17.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src_thrift diff --git ql/src/test/results/clientpositive/input18.q.out ql/src/test/results/clientpositive/input18.q.out index b341510502..4eb75ff55e 100644 --- ql/src/test/results/clientpositive/input18.q.out +++ ql/src/test/results/clientpositive/input18.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input1_limit.q.out ql/src/test/results/clientpositive/input1_limit.q.out index 0ca1552ef2..649e8fb94b 100644 --- ql/src/test/results/clientpositive/input1_limit.q.out +++ ql/src/test/results/clientpositive/input1_limit.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -145,7 +146,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 LIMIT 10 diff --git ql/src/test/results/clientpositive/input20.q.out ql/src/test/results/clientpositive/input20.q.out index 9587445f82..051e94ecf3 100644 --- ql/src/test/results/clientpositive/input20.q.out +++ ql/src/test/results/clientpositive/input20.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input30.q.out ql/src/test/results/clientpositive/input30.q.out index 478cea1861..90c9732117 100644 --- ql/src/test/results/clientpositive/input30.q.out +++ ql/src/test/results/clientpositive/input30.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.tst_dest30 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest30 select count(1) from src @@ -159,7 +160,8 @@ STAGE PLANS: name: default.dest30 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest30 select count(1) from src diff --git ql/src/test/results/clientpositive/input31.q.out ql/src/test/results/clientpositive/input31.q.out index ea2c8f95b3..d3c2c6af09 100644 --- ql/src/test/results/clientpositive/input31.q.out +++ ql/src/test/results/clientpositive/input31.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.tst_dest31 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest31 select count(1) from srcbucket diff --git ql/src/test/results/clientpositive/input32.q.out ql/src/test/results/clientpositive/input32.q.out index d3426a8dfa..fa3b731a78 100644 --- ql/src/test/results/clientpositive/input32.q.out +++ ql/src/test/results/clientpositive/input32.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.tst_dest32 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest32 select count(1) from srcbucket diff --git ql/src/test/results/clientpositive/input33.q.out ql/src/test/results/clientpositive/input33.q.out index 4be5fc5d61..c24ac6e41e 100644 --- ql/src/test/results/clientpositive/input33.q.out +++ ql/src/test/results/clientpositive/input33.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input34.q.out ql/src/test/results/clientpositive/input34.q.out index 72f66c3f17..841f0af16f 100644 --- ql/src/test/results/clientpositive/input34.q.out +++ ql/src/test/results/clientpositive/input34.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input35.q.out ql/src/test/results/clientpositive/input35.q.out index 8b869918e1..76a014eed9 100644 --- ql/src/test/results/clientpositive/input35.q.out +++ ql/src/test/results/clientpositive/input35.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input36.q.out ql/src/test/results/clientpositive/input36.q.out index 76921ad6b8..f75f8fda5a 100644 --- ql/src/test/results/clientpositive/input36.q.out +++ ql/src/test/results/clientpositive/input36.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input38.q.out ql/src/test/results/clientpositive/input38.q.out index 0c4e81d0ef..41a0dbd420 100644 --- ql/src/test/results/clientpositive/input38.q.out +++ ql/src/test/results/clientpositive/input38.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input3_limit.q.out ql/src/test/results/clientpositive/input3_limit.q.out index 560de96131..6c8376861e 100644 --- ql/src/test/results/clientpositive/input3_limit.q.out +++ ql/src/test/results/clientpositive/input3_limit.q.out @@ -107,7 +107,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM (SELECT * FROM T1 DISTRIBUTE BY key) T ORDER BY key, value LIMIT 20 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/input4.q.out ql/src/test/results/clientpositive/input4.q.out index 69843183a5..151aa0c4c1 100644 --- ql/src/test/results/clientpositive/input4.q.out +++ ql/src/test/results/clientpositive/input4.q.out @@ -28,7 +28,8 @@ STAGE PLANS: name: default.input4 Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE INPUT4 PREHOOK: type: LOAD diff --git ql/src/test/results/clientpositive/input5.q.out ql/src/test/results/clientpositive/input5.q.out index a39952878d..75a827c5da 100644 --- ql/src/test/results/clientpositive/input5.q.out +++ ql/src/test/results/clientpositive/input5.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src_thrift diff --git ql/src/test/results/clientpositive/input6.q.out ql/src/test/results/clientpositive/input6.q.out index 3d1a815cf6..35e746ae8c 100644 --- ql/src/test/results/clientpositive/input6.q.out +++ ql/src/test/results/clientpositive/input6.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input7.q.out ql/src/test/results/clientpositive/input7.q.out index 0545b1f774..281f6d4a09 100644 --- ql/src/test/results/clientpositive/input7.q.out +++ ql/src/test/results/clientpositive/input7.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input8.q.out ql/src/test/results/clientpositive/input8.q.out index d76fc2bc02..e49c7e17f7 100644 --- ql/src/test/results/clientpositive/input8.q.out +++ ql/src/test/results/clientpositive/input8.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input9.q.out ql/src/test/results/clientpositive/input9.q.out index af752e0af9..8df7fb760e 100644 --- ql/src/test/results/clientpositive/input9.q.out +++ ql/src/test/results/clientpositive/input9.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_columnarserde.q.out ql/src/test/results/clientpositive/input_columnarserde.q.out index afa0e28d17..95d0e6e7ec 100644 --- ql/src/test/results/clientpositive/input_columnarserde.q.out +++ ql/src/test/results/clientpositive/input_columnarserde.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.input_columnarserde Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE input_columnarserde SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/input_dynamicserde.q.out ql/src/test/results/clientpositive/input_dynamicserde.q.out index 30493be357..a117873164 100644 --- ql/src/test/results/clientpositive/input_dynamicserde.q.out +++ ql/src/test/results/clientpositive/input_dynamicserde.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_lazyserde.q.out ql/src/test/results/clientpositive/input_lazyserde.q.out index 64dc6c14a4..473b5fe4e6 100644 --- ql/src/test/results/clientpositive/input_lazyserde.q.out +++ ql/src/test/results/clientpositive/input_lazyserde.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/input_part1.q.out ql/src/test/results/clientpositive/input_part1.q.out index a68544671f..9a9b1921c9 100644 --- ql/src/test/results/clientpositive/input_part1.q.out +++ ql/src/test/results/clientpositive/input_part1.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/input_part10.q.out ql/src/test/results/clientpositive/input_part10.q.out index 4b552badc6..bb35474df2 100644 --- ql/src/test/results/clientpositive/input_part10.q.out +++ ql/src/test/results/clientpositive/input_part10.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.part_special Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') SELECT 1, 2 FROM src LIMIT 1 diff --git ql/src/test/results/clientpositive/input_part2.q.out ql/src/test/results/clientpositive/input_part2.q.out index d5524feabc..70ed0f641f 100644 --- ql/src/test/results/clientpositive/input_part2.q.out +++ ql/src/test/results/clientpositive/input_part2.q.out @@ -270,7 +270,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -491,7 +492,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 diff --git ql/src/test/results/clientpositive/input_part5.q.out ql/src/test/results/clientpositive/input_part5.q.out index c6ae2fd58d..7da77fb3cd 100644 --- ql/src/test/results/clientpositive/input_part5.q.out +++ ql/src/test/results/clientpositive/input_part5.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_testsequencefile.q.out ql/src/test/results/clientpositive/input_testsequencefile.q.out index 60aaf83df5..c248d03bfc 100644 --- ql/src/test/results/clientpositive/input_testsequencefile.q.out +++ ql/src/test/results/clientpositive/input_testsequencefile.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest4_sequencefile Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_testxpath.q.out ql/src/test/results/clientpositive/input_testxpath.q.out index e07628aaea..a68a5003fe 100644 --- ql/src/test/results/clientpositive/input_testxpath.q.out +++ ql/src/test/results/clientpositive/input_testxpath.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_testxpath2.q.out ql/src/test/results/clientpositive/input_testxpath2.q.out index a0baccf925..ed45157141 100644 --- ql/src/test/results/clientpositive/input_testxpath2.q.out +++ ql/src/test/results/clientpositive/input_testxpath2.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert1.q.out ql/src/test/results/clientpositive/insert1.q.out index 39525787c9..7465e30c46 100644 --- ql/src/test/results/clientpositive/insert1.q.out +++ ql/src/test/results/clientpositive/insert1.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -170,7 +171,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -273,7 +275,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -362,7 +365,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -479,7 +483,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -531,7 +536,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-10 Map Reduce diff --git ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out index 49c1269cc1..a9378f81e5 100644 --- ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out +++ ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out @@ -90,7 +90,8 @@ STAGE PLANS: name: default.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5 @@ -220,7 +221,8 @@ STAGE PLANS: name: default.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out index b5f28d289a..c8bfdc6bb2 100644 --- ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out +++ ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out @@ -100,7 +100,8 @@ STAGE PLANS: name: db2.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11') SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5 @@ -188,7 +189,8 @@ STAGE PLANS: name: db2.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11') SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5 diff --git ql/src/test/results/clientpositive/insert_into1.q.out ql/src/test/results/clientpositive/insert_into1.q.out index da863a7185..3d1438a037 100644 --- ql/src/test/results/clientpositive/insert_into1.q.out +++ ql/src/test/results/clientpositive/insert_into1.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -176,7 +177,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -284,7 +286,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 10 PREHOOK: type: QUERY @@ -389,7 +392,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -486,7 +490,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_into2.q.out ql/src/test/results/clientpositive/insert_into2.q.out index 46fab7b416..90b409cbb1 100644 --- ql/src/test/results/clientpositive/insert_into2.q.out +++ ql/src/test/results/clientpositive/insert_into2.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into2 PARTITION (ds='1') SELECT * FROM src order by key limit 100 PREHOOK: type: QUERY @@ -223,7 +224,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 100 @@ -341,7 +343,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 50 diff --git ql/src/test/results/clientpositive/insert_into3.q.out ql/src/test/results/clientpositive/insert_into3.q.out index ae7523b217..4bda7d9514 100644 --- ql/src/test/results/clientpositive/insert_into3.q.out +++ ql/src/test/results/clientpositive/insert_into3.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -137,7 +138,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 @@ -259,7 +261,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -302,7 +305,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 diff --git ql/src/test/results/clientpositive/insert_into4.q.out ql/src/test/results/clientpositive/insert_into4.q.out index bb4e5571a6..931ae3d7be 100644 --- ql/src/test/results/clientpositive/insert_into4.q.out +++ ql/src/test/results/clientpositive/insert_into4.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.insert_into4a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY @@ -167,7 +168,8 @@ STAGE PLANS: name: default.insert_into4a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY @@ -246,7 +248,8 @@ STAGE PLANS: name: default.insert_into4b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_into5.q.out ql/src/test/results/clientpositive/insert_into5.q.out index 7b471f4433..5e52e49263 100644 --- ql/src/test/results/clientpositive/insert_into5.q.out +++ ql/src/test/results/clientpositive/insert_into5.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.insert_into5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 PREHOOK: type: QUERY @@ -157,7 +158,8 @@ STAGE PLANS: name: default.insert_into5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -270,7 +272,8 @@ STAGE PLANS: name: default.insert_into5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -385,7 +388,8 @@ STAGE PLANS: name: default.insert_into5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_into6.q.out ql/src/test/results/clientpositive/insert_into6.q.out index d93a167a74..964e259771 100644 --- ql/src/test/results/clientpositive/insert_into6.q.out +++ ql/src/test/results/clientpositive/insert_into6.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.insert_into6a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='1') SELECT * FROM src LIMIT 150 PREHOOK: type: QUERY @@ -183,7 +184,8 @@ STAGE PLANS: name: default.insert_into6b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out index 6df425fcec..064a70cefc 100644 --- ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out +++ ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out @@ -364,7 +364,7 @@ Table Parameters: numFiles 1 numRows 0 rawDataSize 0 - totalSize 1554 + totalSize 1555 transactional true #### A masked pattern was here #### @@ -392,9 +392,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 1 Data size: 1554 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1555 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 1554 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1555 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash @@ -486,7 +486,7 @@ Table Parameters: numFiles 2 numRows 0 rawDataSize 0 - totalSize 3109 + totalSize 3110 transactional true #### A masked pattern was here #### @@ -514,9 +514,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 1 Data size: 3109 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 3110 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 3109 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 3110 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash @@ -606,7 +606,7 @@ Table Parameters: numFiles 3 numRows 0 rawDataSize 0 - totalSize 298592 + totalSize 298593 transactional true #### A masked pattern was here #### @@ -634,9 +634,9 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_ivot - Statistics: Num rows: 1 Data size: 298592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 298593 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 1 Data size: 298592 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 298593 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash diff --git ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out index c2732c8896..4c020fd26e 100644 --- ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out +++ ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out @@ -142,7 +142,8 @@ STAGE PLANS: name: default.temp1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TABLE temp2 ( @@ -221,7 +222,8 @@ STAGE PLANS: name: default.temp2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: select * from bucketoutput1 a join bucketoutput2 b on (a.data=b.data) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index 66e42f1992..10b4e1fd8b 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/join17.q.out ql/src/test/results/clientpositive/join17.q.out index f9edc792eb..6c3e5fe40f 100644 --- ql/src/test/results/clientpositive/join17.q.out +++ ql/src/test/results/clientpositive/join17.q.out @@ -192,7 +192,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/join2.q.out ql/src/test/results/clientpositive/join2.q.out index e3d26a29a6..b2067421ec 100644 --- ql/src/test/results/clientpositive/join2.q.out +++ ql/src/test/results/clientpositive/join2.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/join25.q.out ql/src/test/results/clientpositive/join25.q.out index 5ad95c507b..8ed420bbaa 100644 --- ql/src/test/results/clientpositive/join25.q.out +++ ql/src/test/results/clientpositive/join25.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/join26.q.out ql/src/test/results/clientpositive/join26.q.out index b41fd8efe1..134aa523d8 100644 --- ql/src/test/results/clientpositive/join26.q.out +++ ql/src/test/results/clientpositive/join26.q.out @@ -313,7 +313,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join27.q.out ql/src/test/results/clientpositive/join27.q.out index 8b43f3f32a..2e78d2ad22 100644 --- ql/src/test/results/clientpositive/join27.q.out +++ ql/src/test/results/clientpositive/join27.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/join28.q.out ql/src/test/results/clientpositive/join28.q.out index 309bdcd200..c4b122fcb6 100644 --- ql/src/test/results/clientpositive/join28.q.out +++ ql/src/test/results/clientpositive/join28.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq.key1, z.value diff --git ql/src/test/results/clientpositive/join29.q.out ql/src/test/results/clientpositive/join29.q.out index ef02385fe1..d394eda3d4 100644 --- ql/src/test/results/clientpositive/join29.q.out +++ ql/src/test/results/clientpositive/join29.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-9 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join3.q.out ql/src/test/results/clientpositive/join3.q.out index fb378f403a..8c4cfe2c37 100644 --- ql/src/test/results/clientpositive/join3.q.out +++ ql/src/test/results/clientpositive/join3.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/join30.q.out ql/src/test/results/clientpositive/join30.q.out index f06c70ae40..cdd54458db 100644 --- ql/src/test/results/clientpositive/join30.q.out +++ ql/src/test/results/clientpositive/join30.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/join31.q.out ql/src/test/results/clientpositive/join31.q.out index e055b5d2fa..7b0cbdc64e 100644 --- ql/src/test/results/clientpositive/join31.q.out +++ ql/src/test/results/clientpositive/join31.q.out @@ -145,7 +145,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq1.key, count(1) as cnt diff --git ql/src/test/results/clientpositive/join32.q.out ql/src/test/results/clientpositive/join32.q.out index 176989caff..30a5ba9195 100644 --- ql/src/test/results/clientpositive/join32.q.out +++ ql/src/test/results/clientpositive/join32.q.out @@ -366,7 +366,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join33.q.out ql/src/test/results/clientpositive/join33.q.out index 176989caff..30a5ba9195 100644 --- ql/src/test/results/clientpositive/join33.q.out +++ ql/src/test/results/clientpositive/join33.q.out @@ -366,7 +366,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join34.q.out ql/src/test/results/clientpositive/join34.q.out index 67599bc991..072a20b62d 100644 --- ql/src/test/results/clientpositive/join34.q.out +++ ql/src/test/results/clientpositive/join34.q.out @@ -309,7 +309,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join35.q.out ql/src/test/results/clientpositive/join35.q.out index ade6646caa..f8fd4387a9 100644 --- ql/src/test/results/clientpositive/join35.q.out +++ ql/src/test/results/clientpositive/join35.q.out @@ -393,7 +393,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 diff --git ql/src/test/results/clientpositive/join36.q.out ql/src/test/results/clientpositive/join36.q.out index 43a091f629..0fd4d918e6 100644 --- ql/src/test/results/clientpositive/join36.q.out +++ ql/src/test/results/clientpositive/join36.q.out @@ -133,7 +133,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt diff --git ql/src/test/results/clientpositive/join37.q.out ql/src/test/results/clientpositive/join37.q.out index b0a2ee3dfa..c8bef838f4 100644 --- ql/src/test/results/clientpositive/join37.q.out +++ ql/src/test/results/clientpositive/join37.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/join39.q.out ql/src/test/results/clientpositive/join39.q.out index c656762675..93ebdb85c7 100644 --- ql/src/test/results/clientpositive/join39.q.out +++ ql/src/test/results/clientpositive/join39.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value diff --git ql/src/test/results/clientpositive/join4.q.out ql/src/test/results/clientpositive/join4.q.out index 10b7fbd965..cb6bdfe3eb 100644 --- ql/src/test/results/clientpositive/join4.q.out +++ ql/src/test/results/clientpositive/join4.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join5.q.out ql/src/test/results/clientpositive/join5.q.out index 105a731526..87bb50cce7 100644 --- ql/src/test/results/clientpositive/join5.q.out +++ ql/src/test/results/clientpositive/join5.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join6.q.out ql/src/test/results/clientpositive/join6.q.out index 77c8c3a322..724e3bd839 100644 --- ql/src/test/results/clientpositive/join6.q.out +++ ql/src/test/results/clientpositive/join6.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join7.q.out ql/src/test/results/clientpositive/join7.q.out index fec67b2651..6c9be33712 100644 --- ql/src/test/results/clientpositive/join7.q.out +++ ql/src/test/results/clientpositive/join7.q.out @@ -138,7 +138,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join8.q.out ql/src/test/results/clientpositive/join8.q.out index 185b5421a3..6b066d1f57 100644 --- ql/src/test/results/clientpositive/join8.q.out +++ ql/src/test/results/clientpositive/join8.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join9.q.out ql/src/test/results/clientpositive/join9.q.out index a96f341c78..58f8d7dfbe 100644 --- ql/src/test/results/clientpositive/join9.q.out +++ ql/src/test/results/clientpositive/join9.q.out @@ -240,7 +240,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/join_map_ppr.q.out ql/src/test/results/clientpositive/join_map_ppr.q.out index a4d414089e..6eb9889f90 100644 --- ql/src/test/results/clientpositive/join_map_ppr.q.out +++ ql/src/test/results/clientpositive/join_map_ppr.q.out @@ -225,7 +225,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-5 @@ -803,7 +804,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-5 diff --git ql/src/test/results/clientpositive/limit_pushdown_negative.q.out ql/src/test/results/clientpositive/limit_pushdown_negative.q.out index eeb47f7fe5..7d0abfdf97 100644 --- ql/src/test/results/clientpositive/limit_pushdown_negative.q.out +++ ql/src/test/results/clientpositive/limit_pushdown_negative.q.out @@ -296,7 +296,8 @@ STAGE PLANS: name: default.dest_2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -339,5 +340,6 @@ STAGE PLANS: name: default.dest_3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/lineage1.q.out ql/src/test/results/clientpositive/lineage1.q.out index 6c8a22fa9c..7d6af3e1df 100644 --- ql/src/test/results/clientpositive/lineage1.q.out +++ ql/src/test/results/clientpositive/lineage1.q.out @@ -147,7 +147,8 @@ STAGE PLANS: name: default.dest_l1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/list_bucket_dml_1.q.out ql/src/test/results/clientpositive/list_bucket_dml_1.q.out index f70f8b2b9a..ca1cf4f48f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_1.q.out @@ -197,7 +197,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08' diff --git ql/src/test/results/clientpositive/list_bucket_dml_11.q.out ql/src/test/results/clientpositive/list_bucket_dml_11.q.out index 6920dcc7ca..35a33decf6 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_11.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_11.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/list_bucket_dml_12.q.out ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index 55acbb7f5a..af50edfff5 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.list_bucketing_mul_col Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/list_bucket_dml_13.q.out ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index 3a1d2a436a..244724a033 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.list_bucketing_mul_col Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99') diff --git ql/src/test/results/clientpositive/list_bucket_dml_14.q.out ql/src/test/results/clientpositive/list_bucket_dml_14.q.out index f827991243..1e8f518a5f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_14.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_14.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.list_bucketing Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing select * from src diff --git ql/src/test/results/clientpositive/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index 01bc19cbbc..fd6a59125a 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/list_bucket_dml_3.q.out ql/src/test/results/clientpositive/list_bucket_dml_3.q.out index 52646a2609..3386ed9b20 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_3.q.out @@ -191,7 +191,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds='2008-04-08', hr='11') select key, value from srcpart where ds='2008-04-08' diff --git ql/src/test/results/clientpositive/list_bucket_dml_4.q.out ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index 84ada2bd49..e1882d7e70 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') @@ -471,7 +472,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/list_bucket_dml_5.q.out ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index 71d7e16409..1adbb8148e 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -197,7 +197,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08' diff --git ql/src/test/results/clientpositive/list_bucket_dml_6.q.out ql/src/test/results/clientpositive/list_bucket_dml_6.q.out index 8e20ff1301..a3f2837613 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_6.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_6.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr) @@ -520,7 +521,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/list_bucket_dml_7.q.out ql/src/test/results/clientpositive/list_bucket_dml_7.q.out index 58bf240f1b..7db0aa983b 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_7.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_7.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr) @@ -520,7 +521,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/list_bucket_dml_8.q.out ql/src/test/results/clientpositive/list_bucket_dml_8.q.out index b38d332e09..1129544f74 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_8.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_8.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr) diff --git ql/src/test/results/clientpositive/list_bucket_dml_9.q.out ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index 624e3aca41..ae313ed041 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') @@ -471,7 +472,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out index 34dd4877e8..bd02c52ffd 100644 --- ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out +++ ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out @@ -168,7 +168,8 @@ STAGE PLANS: name: default.srcpart_acid Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -346,7 +347,8 @@ STAGE PLANS: name: default.srcpart_acid Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acid where key in( '1001', '213', '43') PREHOOK: type: QUERY @@ -662,7 +664,8 @@ STAGE PLANS: name: default.srcpart_acidb Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -841,7 +844,8 @@ STAGE PLANS: name: default.srcpart_acidb Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acidb where key in( '1001', '213', '43') PREHOOK: type: QUERY @@ -1156,7 +1160,8 @@ STAGE PLANS: name: default.srcpart_acidv Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -1334,7 +1339,8 @@ STAGE PLANS: name: default.srcpart_acidv Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acidv where key in( '1001', '213', '43') PREHOOK: type: QUERY @@ -1650,7 +1656,8 @@ STAGE PLANS: name: default.srcpart_acidvb Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -1829,7 +1836,8 @@ STAGE PLANS: name: default.srcpart_acidvb Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acidvb where key in( '1001', '213', '43') PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out index 6369dd3477..2926cef4fa 100644 --- ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out +++ ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out @@ -380,6 +380,46 @@ POSTHOOK: query: create table if not exists nzhang_part14 (key string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: desc formatted nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment + +key string + +# Partition Information +# col_name data_type comment + +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a @@ -405,6 +445,44 @@ POSTHOOK: Output: default@nzhang_part14@value= POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ POSTHOOK: Lineage: nzhang_part14 PARTITION(value= ).key EXPRESSION [] POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] +PREHOOK: query: desc formatted nzhang_part14 partition (value=' ') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 partition (value=' ') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment + +key string + +# Partition Information +# col_name data_type comment + +value string + +# Detailed Partition Information +Partition Value: [ ] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 4 + totalSize 6 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain select key from nzhang_part14 PREHOOK: type: QUERY POSTHOOK: query: explain select key from nzhang_part14 diff --git ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out new file mode 100644 index 0000000000..6b730c2bfd --- /dev/null +++ ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out @@ -0,0 +1,452 @@ +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int 1 1 0 1 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int 1 2 0 2 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment + +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +insert_num int from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector + +c1 string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} diff --git ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out index a61eafadc9..25cd3756d5 100644 --- ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out +++ ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out @@ -687,7 +687,6 @@ Database: default Table: alter5 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"col1\":\"true\"}} numFiles 1 totalSize 1906 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/auto_join1.q.out ql/src/test/results/clientpositive/llap/auto_join1.q.out index 5329f84f34..bbe63e24d4 100644 --- ql/src/test/results/clientpositive/llap/auto_join1.q.out +++ ql/src/test/results/clientpositive/llap/auto_join1.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index c6b14abd2f..80a74fbfdf 100644 --- ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -1612,7 +1612,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1625,7 +1626,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -1850,7 +1852,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1863,7 +1866,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out index f358a594b4..a6d73097e0 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -178,7 +179,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -366,7 +368,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -379,7 +382,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -567,7 +571,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -580,7 +585,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 diff --git ql/src/test/results/clientpositive/llap/bucket2.q.out ql/src/test/results/clientpositive/llap/bucket2.q.out index e0c92ced9f..9954c2dd99 100644 --- ql/src/test/results/clientpositive/llap/bucket2.q.out +++ ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -174,7 +174,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/llap/bucket3.q.out ql/src/test/results/clientpositive/llap/bucket3.q.out index 8e6d85ca80..218f9b7e48 100644 --- ql/src/test/results/clientpositive/llap/bucket3.q.out +++ ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -171,7 +171,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') diff --git ql/src/test/results/clientpositive/llap/bucket4.q.out ql/src/test/results/clientpositive/llap/bucket4.q.out index 5fbffc96a9..2115565096 100644 --- ql/src/test/results/clientpositive/llap/bucket4.q.out +++ ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -177,7 +177,8 @@ STAGE PLANS: name: default.bucket4_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket4_1 diff --git ql/src/test/results/clientpositive/llap/bucket5.q.out ql/src/test/results/clientpositive/llap/bucket5.q.out index 0b5a14d6ff..6ca04ee4a0 100644 --- ql/src/test/results/clientpositive/llap/bucket5.q.out +++ ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -248,7 +248,8 @@ STAGE PLANS: name: default.bucketed_table Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -280,7 +281,8 @@ STAGE PLANS: name: default.unbucketed_table Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 diff --git ql/src/test/results/clientpositive/llap/bucket6.q.out ql/src/test/results/clientpositive/llap/bucket6.q.out index 20895f8a9f..42f062b0f6 100644 --- ql/src/test/results/clientpositive/llap/bucket6.q.out +++ ql/src/test/results/clientpositive/llap/bucket6.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.src_bucket Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table src_bucket select key,value from srcpart PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/bucket_many.q.out ql/src/test/results/clientpositive/llap/bucket_many.q.out index b78cbaa0c3..4f3bee20ce 100644 --- ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -174,7 +174,8 @@ STAGE PLANS: name: default.bucket_many Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket_many diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out index e1373c438a..c6e00d5c16 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out @@ -638,7 +638,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -1004,7 +1005,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out index c254d89dfa..57eead0ed5 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out @@ -352,7 +352,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -724,7 +725,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -1162,7 +1164,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out index 167813553f..184e8905f8 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out @@ -376,7 +376,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -748,7 +749,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out index dd67a55a72..388661e012 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out @@ -372,7 +372,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -728,7 +729,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out index 13eecea578..b907c2dbd8 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out @@ -189,7 +189,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -368,7 +369,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -571,7 +573,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -780,7 +783,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -971,7 +975,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.v1, b.v2) @@ -1162,7 +1167,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key+a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out index 9959ba4738..f5f5f91e82 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out @@ -167,7 +167,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key2, concat(a.value, b.value) @@ -353,7 +354,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq1.key, subq1.key2, subq1.value from @@ -539,7 +541,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -660,7 +663,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -787,7 +791,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.key, subq2.key2, subq2.value from @@ -991,7 +996,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.k2, subq2.k1, subq2.value from @@ -1205,5 +1211,6 @@ STAGE PLANS: name: default.test_table4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out index 62ed4c8a64..66693a3c93 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out @@ -167,7 +167,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -350,7 +351,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -539,7 +541,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/llap/column_table_stats.q.out ql/src/test/results/clientpositive/llap/column_table_stats.q.out index c397f80e71..00094319a7 100644 --- ql/src/test/results/clientpositive/llap/column_table_stats.q.out +++ ql/src/test/results/clientpositive/llap/column_table_stats.q.out @@ -59,7 +59,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -176,11 +175,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.s/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.s/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -318,7 +315,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -494,11 +490,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.spart/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.spart/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -735,7 +729,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -911,11 +904,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.spart/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.spart/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -1152,7 +1143,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -1280,11 +1270,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.spart/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.spart/ Column Stats Desc: Columns: key, value Column Types: string, string diff --git ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out index c7199760ac..dc620f43b6 100644 --- ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out +++ ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out @@ -60,7 +60,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -179,10 +178,8 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: key, value Column Types: string, string @@ -315,7 +312,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -485,10 +481,8 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: key, value Column Types: string, string @@ -720,7 +714,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -845,10 +838,8 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: key, value Column Types: string, string diff --git ql/src/test/results/clientpositive/llap/ctas.q.out ql/src/test/results/clientpositive/llap/ctas.q.out index fe492e43ad..23514de58d 100644 --- ql/src/test/results/clientpositive/llap/ctas.q.out +++ ql/src/test/results/clientpositive/llap/ctas.q.out @@ -97,7 +97,8 @@ STAGE PLANS: name: default.nzhang_CTAS1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -253,7 +254,8 @@ STAGE PLANS: name: default.nzhang_ctas2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -409,7 +411,8 @@ STAGE PLANS: name: default.nzhang_ctas3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -630,7 +633,8 @@ STAGE PLANS: name: default.nzhang_ctas4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -790,7 +794,8 @@ STAGE PLANS: name: default.nzhang_ctas5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index e31ebaf5d3..acd607628a 100644 --- ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -174,7 +174,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index ffc39575d1..8e241d9aa6 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out index adc24158be..3962b29037 100644 --- ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out @@ -258,7 +258,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -271,7 +272,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -284,7 +286,8 @@ STAGE PLANS: name: default.merge_tmp_table Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -297,7 +300,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) @@ -429,7 +433,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain merge into acidTbl as t using ( select * from nonAcidOrcTbl where a > 0 @@ -729,7 +734,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -742,7 +748,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -755,7 +762,8 @@ STAGE PLANS: name: default.merge_tmp_table Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -768,7 +776,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop database if exists type2_scd_helper cascade PREHOOK: type: DROPDATABASE diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index 4a79503587..62a38b946a 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -144,7 +144,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -382,7 +383,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -624,7 +626,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -860,7 +863,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -1157,7 +1161,8 @@ STAGE PLANS: name: default.ss_part_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part_orc partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -1393,7 +1398,8 @@ STAGE PLANS: name: default.ss_part_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part_orc partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -1671,7 +1677,8 @@ STAGE PLANS: name: default.hive13_dp1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table `hive13_dp1` partition(`day`) select @@ -1801,7 +1808,8 @@ STAGE PLANS: name: default.hive13_dp1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table `hive13_dp1` partition(`day`) select diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 6488e9ee60..fb653efa03 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -140,7 +140,8 @@ STAGE PLANS: name: default.acid_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part set value = 'bar' where key = 'foo' and ds='2008-04-08' PREHOOK: type: QUERY @@ -234,7 +235,8 @@ STAGE PLANS: name: default.acid_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part set value = 'bar' where key = 'foo' and ds in ('2008-04-08') PREHOOK: type: QUERY @@ -420,7 +422,8 @@ STAGE PLANS: name: default.acid_part_sdpo Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' PREHOOK: type: QUERY @@ -514,7 +517,8 @@ STAGE PLANS: name: default.acid_part_sdpo Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part_sdpo set value = 'bar' where key = 'foo' and ds in ('2008-04-08') PREHOOK: type: QUERY @@ -710,7 +714,8 @@ STAGE PLANS: name: default.acid_2l_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 PREHOOK: type: QUERY @@ -805,7 +810,8 @@ STAGE PLANS: name: default.acid_2l_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 PREHOOK: type: QUERY @@ -929,7 +935,8 @@ STAGE PLANS: name: default.acid_2l_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from acid_2L_part where value = 'bar' PREHOOK: type: QUERY @@ -1122,7 +1129,8 @@ STAGE PLANS: name: default.acid_2l_part_sdpo Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 PREHOOK: type: QUERY @@ -1217,7 +1225,8 @@ STAGE PLANS: name: default.acid_2l_part_sdpo Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 PREHOOK: type: QUERY @@ -1341,7 +1350,8 @@ STAGE PLANS: name: default.acid_2l_part_sdpo Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from acid_2L_part_sdpo where value = 'bar' PREHOOK: type: QUERY @@ -1536,7 +1546,8 @@ STAGE PLANS: name: default.acid_2l_part_sdpo_no_cp Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 PREHOOK: type: QUERY @@ -1632,7 +1643,8 @@ STAGE PLANS: name: default.acid_2l_part_sdpo_no_cp Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 48c02a24ad..9a6294fa91 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -46,7 +46,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -81,7 +81,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -3178,7 +3178,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.nzhang_CTAS1 @@ -3231,7 +3231,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.nzhang_ctas3 @@ -4833,7 +4833,7 @@ Reducer 4 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.part_4"} @@ -4884,7 +4884,7 @@ Stage-4 PartitionCols:_col2 Please refer to the previous PTF Operator [PTF_3] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.part_5"} @@ -5247,7 +5247,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest_j1"} diff --git ql/src/test/results/clientpositive/llap/explainuser_2.q.out ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 4693c2b156..a164dd38bc 100644 --- ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -2605,7 +2605,7 @@ Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 11 (SIMPLE_EDGE) Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Union 4 (CONTAINS) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -2838,13 +2838,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_44] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -2908,7 +2908,7 @@ Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) Reducer 8 <- Union 7 (SIMPLE_EDGE) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -3170,13 +3170,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_120] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -3221,7 +3221,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -3276,7 +3276,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_13] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -3395,7 +3395,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -3468,7 +3468,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -3503,7 +3503,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -3562,7 +3562,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} diff --git ql/src/test/results/clientpositive/llap/groupby1.q.out ql/src/test/results/clientpositive/llap/groupby1.q.out index 5917013fa4..d58a9fd69f 100644 --- ql/src/test/results/clientpositive/llap/groupby1.q.out +++ ql/src/test/results/clientpositive/llap/groupby1.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/groupby2.q.out ql/src/test/results/clientpositive/llap/groupby2.q.out index f94db4edf1..5ab62358ed 100644 --- ql/src/test/results/clientpositive/llap/groupby2.q.out +++ ql/src/test/results/clientpositive/llap/groupby2.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/llap/groupby3.q.out ql/src/test/results/clientpositive/llap/groupby3.q.out index 3495de6d3f..897946b4ba 100644 --- ql/src/test/results/clientpositive/llap/groupby3.q.out +++ ql/src/test/results/clientpositive/llap/groupby3.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/llap/insert1.q.out ql/src/test/results/clientpositive/llap/insert1.q.out index af4b8871e1..cd49e28b27 100644 --- ql/src/test/results/clientpositive/llap/insert1.q.out +++ ql/src/test/results/clientpositive/llap/insert1.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY @@ -130,7 +131,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create database x PREHOOK: type: CREATEDATABASE @@ -198,7 +200,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table default.INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY @@ -252,7 +255,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain from insert2 @@ -329,7 +333,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -342,7 +347,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE DATABASE db2 PREHOOK: type: CREATEDATABASE diff --git ql/src/test/results/clientpositive/llap/insert_into1.q.out ql/src/test/results/clientpositive/llap/insert_into1.q.out index df72ca120d..61297f04c8 100644 --- ql/src/test/results/clientpositive/llap/insert_into1.q.out +++ ql/src/test/results/clientpositive/llap/insert_into1.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -204,7 +205,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -326,7 +328,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 10 PREHOOK: type: QUERY @@ -426,7 +429,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table insert_into1 select 1, 'a' PREHOOK: type: QUERY @@ -488,7 +492,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into insert_into1 select 2, 'b' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/insert_into2.q.out ql/src/test/results/clientpositive/llap/insert_into2.q.out index a42c651d63..ef2b875efb 100644 --- ql/src/test/results/clientpositive/llap/insert_into2.q.out +++ ql/src/test/results/clientpositive/llap/insert_into2.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into2 PARTITION (ds='1') SELECT * FROM src order by key limit 100 PREHOOK: type: QUERY @@ -251,7 +252,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 100 @@ -383,7 +385,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 50 diff --git ql/src/test/results/clientpositive/llap/join1.q.out ql/src/test/results/clientpositive/llap/join1.q.out index 724481beef..661f55caaf 100644 --- ql/src/test/results/clientpositive/llap/join1.q.out +++ ql/src/test/results/clientpositive/llap/join1.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/llap/join32_lessSize.q.out ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index d1c396a71c..ddc45a52eb 100644 --- ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -364,7 +364,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -932,7 +933,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -1403,7 +1405,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -1881,7 +1884,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -2150,7 +2154,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, x.value, res.value @@ -2406,7 +2411,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, y.value, res.value diff --git ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index cdb688b139..3b8feb50a0 100644 --- ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/llap/llap_stats.q.out ql/src/test/results/clientpositive/llap/llap_stats.q.out index 4a48627597..299d9b4dbb 100644 --- ql/src/test/results/clientpositive/llap/llap_stats.q.out +++ ql/src/test/results/clientpositive/llap/llap_stats.q.out @@ -94,7 +94,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -149,10 +148,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: ctinyint, csmallint Column Types: tinyint, smallint diff --git ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out index 6ad9af8086..73293738ee 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -136,7 +137,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out index ce5517a54d..d1906c8c32 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out @@ -97,7 +97,8 @@ STAGE PLANS: name: default.nzhang_part_bucket Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part_bucket partition (ds='2010-03-23', hr) select key, value, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out index b34975fedc..7a73dae17d 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.nzhang_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part3 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index 9c4e8a891b..f6bc531fec 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.nzhang_part5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part5 partition (value) select key, value from src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/mapreduce1.q.out ql/src/test/results/clientpositive/llap/mapreduce1.q.out index 37f92d9f20..0e94e71d27 100644 --- ql/src/test/results/clientpositive/llap/mapreduce1.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce1.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/llap/mapreduce2.q.out ql/src/test/results/clientpositive/llap/mapreduce2.q.out index 71bbb7e612..6485f587f8 100644 --- ql/src/test/results/clientpositive/llap/mapreduce2.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce2.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/llap/merge1.q.out ql/src/test/results/clientpositive/llap/merge1.q.out index ec794dc1ea..e77a5dfe21 100644 --- ql/src/test/results/clientpositive/llap/merge1.q.out +++ ql/src/test/results/clientpositive/llap/merge1.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest1 select key, count(1) from src group by key @@ -513,7 +514,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest1 select key from test_src PREHOOK: type: QUERY @@ -579,7 +581,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest1 select key from test_src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/merge2.q.out ql/src/test/results/clientpositive/llap/merge2.q.out index 6ad6864eb3..25b194697a 100644 --- ql/src/test/results/clientpositive/llap/merge2.q.out +++ ql/src/test/results/clientpositive/llap/merge2.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table test1 select key, count(1) from src group by key @@ -513,7 +514,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table test1 select key from test_src PREHOOK: type: QUERY @@ -579,7 +581,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table test1 select key from test_src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out index 58c29fe667..16628fc6a2 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out @@ -20,7 +20,7 @@ POSTHOOK: Lineage: smalltbl1.value SIMPLE [(src)src.FieldSchema(name:value, type RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: create table smallTbl2(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -43,7 +43,7 @@ POSTHOOK: Lineage: smalltbl2.value SIMPLE [(src)src.FieldSchema(name:value, type RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: create table smallTbl3(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -66,7 +66,7 @@ POSTHOOK: Lineage: smalltbl3.value SIMPLE [(src)src.FieldSchema(name:value, type RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: create table smallTbl4(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -89,7 +89,7 @@ POSTHOOK: Lineage: smalltbl4.value SIMPLE [(src)src.FieldSchema(name:value, type RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: create table bigTbl(key string, value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -156,7 +156,7 @@ POSTHOOK: Lineage: bigtbl.value EXPRESSION [(src)src.FieldSchema(name:value, typ RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: explain select count(*) FROM (select bigTbl.key as key, bigTbl.value as value1, @@ -741,7 +741,7 @@ POSTHOOK: Lineage: bigtbl.value EXPRESSION [(src)src.FieldSchema(name:value, typ RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: EXPLAIN SELECT SUM(HASH(join3.key1)), SUM(HASH(join3.key2)), diff --git ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out index 88951c3d3a..55f8e3a009 100644 --- ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out +++ ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out @@ -1641,7 +1641,7 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=1).value SIMPLE [(src)src.Fi RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: INSERT OVERWRITE TABLE part_table PARTITION (partitionId=2) SELECT key, value FROM src1 ORDER BY key, value PREHOOK: type: QUERY @@ -1657,7 +1657,7 @@ POSTHOOK: Lineage: part_table PARTITION(partitionid=2).value SIMPLE [(src1)src1. RUN: Stage-1:MAPRED RUN: Stage-2:DEPENDENCY_COLLECTION RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: EXPLAIN SELECT count(*) FROM part_table x JOIN src1 y ON (x.key = y.key) diff --git ql/src/test/results/clientpositive/llap/multi_insert.q.out ql/src/test/results/clientpositive/llap/multi_insert.q.out index 58fc759f26..9ac4518be3 100644 --- ql/src/test/results/clientpositive/llap/multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -102,7 +103,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -232,7 +234,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -245,7 +248,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -375,7 +379,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -388,7 +393,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -518,7 +524,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -531,7 +538,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -679,7 +687,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -692,7 +701,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -833,7 +843,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -846,7 +857,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -987,7 +999,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1000,7 +1013,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1141,7 +1155,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1154,7 +1169,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1312,7 +1328,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1325,7 +1342,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1509,7 +1527,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1522,7 +1541,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1706,7 +1726,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1719,7 +1740,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1903,7 +1925,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1916,7 +1939,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 diff --git ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out index 6d20939009..df92c0004f 100644 --- ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -178,7 +179,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C @@ -431,7 +433,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -444,7 +447,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -673,7 +677,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -686,7 +691,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -699,7 +705,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -983,7 +990,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -996,7 +1004,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1009,7 +1018,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C @@ -1354,7 +1364,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1367,7 +1378,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1380,7 +1392,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1393,7 +1406,8 @@ STAGE PLANS: name: default.src_lv4 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key diff --git ql/src/test/results/clientpositive/llap/orc_merge1.q.out ql/src/test/results/clientpositive/llap/orc_merge1.q.out index ba29491001..5669a5baf4 100644 --- ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -191,7 +192,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -320,7 +322,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez diff --git ql/src/test/results/clientpositive/llap/orc_merge10.q.out ql/src/test/results/clientpositive/llap/orc_merge10.q.out index dd5d1cbbe7..4671e477e0 100644 --- ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -191,7 +192,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -320,7 +322,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -459,7 +462,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge2.q.out ql/src/test/results/clientpositive/llap/orc_merge2.q.out index c38852a95b..53a64248a3 100644 --- ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, diff --git ql/src/test/results/clientpositive/llap/orc_merge3.q.out ql/src/test/results/clientpositive/llap/orc_merge3.q.out index c3ca701152..dca37ecd2b 100644 --- ql/src/test/results/clientpositive/llap/orc_merge3.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge3.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b SELECT key, value FROM orcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/orc_merge4.q.out ql/src/test/results/clientpositive/llap/orc_merge4.q.out index a16d2bf11b..a2aa416681 100644 --- ql/src/test/results/clientpositive/llap/orc_merge4.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge4.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b SELECT key, value FROM orcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/orc_merge5.q.out ql/src/test/results/clientpositive/llap/orc_merge5.q.out index 8be0f55cfc..0c37d21196 100644 --- ql/src/test/results/clientpositive/llap/orc_merge5.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge5.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -178,7 +179,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -299,7 +301,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5b concatenate PREHOOK: type: ALTER_TABLE_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge6.q.out ql/src/test/results/clientpositive/llap/orc_merge6.q.out index afba086cde..d1996f6f83 100644 --- ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (year="2000",hour=24) select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -226,7 +227,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -434,7 +436,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(year="2000",hour=24) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge7.q.out ql/src/test/results/clientpositive/llap/orc_merge7.q.out index 58e37c28d0..373266729f 100644 --- ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 PREHOOK: type: QUERY @@ -255,7 +256,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -540,7 +542,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out index ba29491001..5669a5baf4 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -191,7 +192,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -320,7 +322,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out index f3d34cce05..dfd5552ce9 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index 1419f90553..a0dd4a488b 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 order by userid PREHOOK: type: QUERY @@ -286,7 +287,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/parallel.q.out ql/src/test/results/clientpositive/llap/parallel.q.out index aea94170ea..3beb3400d7 100644 --- ql/src/test/results/clientpositive/llap/parallel.q.out +++ ql/src/test/results/clientpositive/llap/parallel.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -134,7 +135,8 @@ STAGE PLANS: name: default.src_b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/llap/parallel_colstats.q.out ql/src/test/results/clientpositive/llap/parallel_colstats.q.out index e89bf2f177..4ac3fbb2f5 100644 --- ql/src/test/results/clientpositive/llap/parallel_colstats.q.out +++ ql/src/test/results/clientpositive/llap/parallel_colstats.q.out @@ -29,8 +29,6 @@ STAGE DEPENDENCIES: Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-4, Stage-5 - Stage-7 depends on stages: Stage-4, Stage-5 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 @@ -181,22 +179,13 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-6 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string Table: default.src_a - Stage: Stage-7 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.src_b - Stage: Stage-1 Move Operator tables: @@ -208,7 +197,12 @@ STAGE PLANS: name: default.src_b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.src_b PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/llap/ptf.q.out ql/src/test/results/clientpositive/llap/ptf.q.out index c69ed1ddc0..4f79e260ae 100644 --- ql/src/test/results/clientpositive/llap/ptf.q.out +++ ql/src/test/results/clientpositive/llap/ptf.q.out @@ -3181,7 +3181,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3194,7 +3195,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part partition by p_mfgr diff --git ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out index 42d3bd0a14..066295f090 100644 --- ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out @@ -97,7 +97,8 @@ STAGE PLANS: name: default.rcfile_createas1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out index 2bb8e8ab1d..e2ecc76948 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.rcfile_merge2a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge2a PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, diff --git ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out index d027bd3ec5..02ee0e0181 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.rcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge3b SELECT key, value FROM rcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out index f4a0353348..52c97d11d5 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.rcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge3b SELECT key, value FROM rcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 40524f1515..65b74ee319 100644 --- ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -175,7 +175,8 @@ STAGE PLANS: name: default.bucket5_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket5_1 @@ -388,6 +389,7 @@ STAGE PLANS: name: default.complex_tbl_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/sample1.q.out ql/src/test/results/clientpositive/llap/sample1.q.out index 1a7fb3254e..5b69bb5832 100644 --- ql/src/test/results/clientpositive/llap/sample1.q.out +++ ql/src/test/results/clientpositive/llap/sample1.q.out @@ -161,7 +161,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/llap/skewjoin.q.out ql/src/test/results/clientpositive/llap/skewjoin.q.out index 5a87f93e41..b31d166179 100644 --- ql/src/test/results/clientpositive/llap/skewjoin.q.out +++ ql/src/test/results/clientpositive/llap/skewjoin.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out index c061df6f72..135f83eca9 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' @@ -281,7 +282,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' and a.key = 238 @@ -405,7 +407,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table2 a WHERE a.ds = '2' diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out index 1f4fcae83c..64763fae40 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out index 91b8f81cb6..07e669fff7 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -1332,7 +1333,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -2530,7 +2532,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 @@ -2644,7 +2647,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 diff --git ql/src/test/results/clientpositive/llap/sqlmerge.q.out ql/src/test/results/clientpositive/llap/sqlmerge.q.out index 61349f8e5b..d6ac04c5d9 100644 --- ql/src/test/results/clientpositive/llap/sqlmerge.q.out +++ ql/src/test/results/clientpositive/llap/sqlmerge.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -241,7 +242,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -254,7 +256,8 @@ STAGE PLANS: name: default.merge_tmp_table Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -267,7 +270,8 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) @@ -369,5 +373,6 @@ STAGE PLANS: name: default.acidtbl Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/stats11.q.out ql/src/test/results/clientpositive/llap/stats11.q.out index 387a454cb9..8107e556f4 100644 --- ql/src/test/results/clientpositive/llap/stats11.q.out +++ ql/src/test/results/clientpositive/llap/stats11.q.out @@ -54,7 +54,8 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') PREHOOK: type: LOAD @@ -552,7 +553,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -918,7 +920,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out index d6eacf675a..9315d6bdde 100644 --- ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out +++ ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out @@ -44,7 +44,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan PREHOOK: type: QUERY @@ -315,7 +316,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/temp_table.q.out ql/src/test/results/clientpositive/llap/temp_table.q.out index 462edae4fe..4cf47b5bd9 100644 --- ql/src/test/results/clientpositive/llap/temp_table.q.out +++ ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -52,7 +52,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -124,7 +125,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/tez_dml.q.out ql/src/test/results/clientpositive/llap/tez_dml.q.out index 84c65a7e66..adccaae2aa 100644 --- ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.tmp_src Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -480,7 +481,8 @@ STAGE PLANS: name: default.tmp_src_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE tmp_src_part PARTITION (d) SELECT * FROM tmp_src PREHOOK: type: QUERY @@ -917,7 +919,8 @@ STAGE PLANS: name: default.even Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -930,7 +933,8 @@ STAGE PLANS: name: default.odd Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT INTO TABLE even SELECT key, value WHERE key % 2 = 0 diff --git ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out index 3e434751b4..c3ec623073 100644 --- ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out @@ -369,7 +369,8 @@ STAGE PLANS: name: default.ct_events1_test Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-0 @@ -1353,7 +1354,8 @@ STAGE PLANS: name: default.ct_events1_test Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-0 diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out index 2ca78d7af8..5dfdede518 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out @@ -122,7 +122,8 @@ STAGE PLANS: name: default.partunion1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table partunion1 partition(part1) select temps.* from ( diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out index 78a17dfbc4..e7599a45f1 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out @@ -144,7 +144,8 @@ STAGE PLANS: name: default.partunion1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez diff --git ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index d399c5e4f5..f9535c50bb 100644 --- ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -237,7 +237,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -250,7 +251,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( select key, value from ( @@ -1144,7 +1146,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1157,7 +1160,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( select key, value from src s0 @@ -2047,7 +2051,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2060,7 +2065,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( select key, value from src s0 @@ -2908,7 +2914,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2921,7 +2928,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION all @@ -3755,7 +3763,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3768,7 +3777,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION distinct diff --git ql/src/test/results/clientpositive/llap/union4.q.out ql/src/test/results/clientpositive/llap/union4.q.out index 796bc60a36..2716072320 100644 --- ql/src/test/results/clientpositive/llap/union4.q.out +++ ql/src/test/results/clientpositive/llap/union4.q.out @@ -134,7 +134,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/llap/union6.q.out ql/src/test/results/clientpositive/llap/union6.q.out index 4043e3cb87..b001433938 100644 --- ql/src/test/results/clientpositive/llap/union6.q.out +++ ql/src/test/results/clientpositive/llap/union6.q.out @@ -107,7 +107,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 diff --git ql/src/test/results/clientpositive/llap/union_fast_stats.q.out ql/src/test/results/clientpositive/llap/union_fast_stats.q.out index b13c0ee464..d1be4e849f 100644 --- ql/src/test/results/clientpositive/llap/union_fast_stats.q.out +++ ql/src/test/results/clientpositive/llap/union_fast_stats.q.out @@ -513,7 +513,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 1069 - totalSize 3245 + totalSize 3243 #### A masked pattern was here #### # Storage Information @@ -566,7 +566,7 @@ Table Parameters: numFiles 1 numRows 15 rawDataSize 3320 - totalSize 3245 + totalSize 3243 #### A masked pattern was here #### # Storage Information @@ -631,7 +631,7 @@ Table Parameters: numFiles 2 numRows 20 rawDataSize 4389 - totalSize 4618 + totalSize 4616 #### A masked pattern was here #### # Storage Information diff --git ql/src/test/results/clientpositive/llap/union_stats.q.out ql/src/test/results/clientpositive/llap/union_stats.q.out index 1f3dc82e15..15d510a573 100644 --- ql/src/test/results/clientpositive/llap/union_stats.q.out +++ ql/src/test/results/clientpositive/llap/union_stats.q.out @@ -202,7 +202,8 @@ STAGE PLANS: name: default.t Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-0 diff --git ql/src/test/results/clientpositive/llap/union_top_level.q.out ql/src/test/results/clientpositive/llap/union_top_level.q.out index cfbc069869..a010232171 100644 --- ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -562,7 +562,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -796,7 +797,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a @@ -1016,7 +1018,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a diff --git ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index 6997af90d3..6c5f539282 100644 --- ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -2031,7 +2031,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2044,7 +2045,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -2302,7 +2304,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2315,7 +2318,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/llap/vector_bucket.q.out ql/src/test/results/clientpositive/llap/vector_bucket.q.out index e6d57d6c15..ed15ff0d89 100644 --- ql/src/test/results/clientpositive/llap/vector_bucket.q.out +++ ql/src/test/results/clientpositive/llap/vector_bucket.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.non_orc_table Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: select a, b from non_orc_table order by a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_char_4.q.out ql/src/test/results/clientpositive/llap/vector_char_4.q.out index 0bf1a40424..52843389da 100644 --- ql/src/test/results/clientpositive/llap/vector_char_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_4.q.out @@ -195,5 +195,6 @@ STAGE PLANS: name: default.char_lazy_binary_columnar Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/vector_complex_all.q.out ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index 5a42218e73..7380194b0d 100644 --- ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -838,7 +838,8 @@ STAGE PLANS: name: default.orc_create_complex Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 4' is a cross product PREHOOK: query: INSERT INTO TABLE orc_create_complex diff --git ql/src/test/results/clientpositive/llap/vector_groupby4.q.out ql/src/test/results/clientpositive/llap/vector_groupby4.q.out index bf2a366284..99ae3e67d9 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby4.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby4.q.out @@ -162,7 +162,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1) diff --git ql/src/test/results/clientpositive/llap/vector_groupby6.q.out ql/src/test/results/clientpositive/llap/vector_groupby6.q.out index 9fa46fb976..45c171b240 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby6.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby6.q.out @@ -162,7 +162,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1) diff --git ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out index 6b2ee48fdf..e2539b646b 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out @@ -884,7 +884,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -897,7 +898,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with cube diff --git ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out index 0e9d6a6c09..bc7b4dbfc8 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out @@ -573,7 +573,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -586,7 +587,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with rollup diff --git ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out index 1c86c0cf26..548fea644f 100644 --- ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.orc_rn1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -182,7 +183,8 @@ STAGE PLANS: name: default.orc_rn2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -195,7 +197,8 @@ STAGE PLANS: name: default.orc_rn3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from orc1 a insert overwrite table orc_rn1 select a.* where a.rn < 100 diff --git ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out index a05e304214..b4a2d3177b 100644 --- ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out @@ -195,5 +195,6 @@ STAGE PLANS: name: default.varchar_lazy_binary_columnar Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out index 7f0b9dae04..70e0d37979 100644 --- ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out +++ ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out @@ -410,7 +410,8 @@ STAGE PLANS: name: default.varchar_3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index d4b9a1cafd..cd6e61cf99 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out index 1b2fc23235..bdd1138a6a 100644 --- ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out @@ -4021,7 +4021,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4034,7 +4035,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part_orc partition by p_mfgr diff --git ql/src/test/results/clientpositive/load_dyn_part1.q.out ql/src/test/results/clientpositive/load_dyn_part1.q.out index 84d806d3a9..72b6e061b4 100644 --- ql/src/test/results/clientpositive/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/load_dyn_part1.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -185,7 +186,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-10 Map Reduce diff --git ql/src/test/results/clientpositive/load_dyn_part10.q.out ql/src/test/results/clientpositive/load_dyn_part10.q.out index 99d357217d..5e932c429b 100644 --- ql/src/test/results/clientpositive/load_dyn_part10.q.out +++ ql/src/test/results/clientpositive/load_dyn_part10.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.nzhang_part10 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part10 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' diff --git ql/src/test/results/clientpositive/load_dyn_part13.q.out ql/src/test/results/clientpositive/load_dyn_part13.q.out index 9e0ac6fee2..e2d43c59f3 100644 --- ql/src/test/results/clientpositive/load_dyn_part13.q.out +++ ql/src/test/results/clientpositive/load_dyn_part13.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.nzhang_part13 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part13 partition (ds="2010-03-03", hr) select * from ( diff --git ql/src/test/results/clientpositive/load_dyn_part14.q.out ql/src/test/results/clientpositive/load_dyn_part14.q.out index a6a5c63cc5..037d949aac 100644 --- ql/src/test/results/clientpositive/load_dyn_part14.q.out +++ ql/src/test/results/clientpositive/load_dyn_part14.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.nzhang_part14 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/load_dyn_part2.q.out ql/src/test/results/clientpositive/load_dyn_part2.q.out index 93778a22e3..da5c8ddfe2 100644 --- ql/src/test/results/clientpositive/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/load_dyn_part2.q.out @@ -83,7 +83,8 @@ STAGE PLANS: name: default.nzhang_part_bucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part_bucket partition (ds='2010-03-23', hr) select key, value, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/load_dyn_part3.q.out ql/src/test/results/clientpositive/load_dyn_part3.q.out index 3849100785..265b40afd9 100644 --- ql/src/test/results/clientpositive/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/load_dyn_part3.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.nzhang_part3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part3 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/load_dyn_part4.q.out ql/src/test/results/clientpositive/load_dyn_part4.q.out index 40b0bbbe8c..2c4e0b39e4 100644 --- ql/src/test/results/clientpositive/load_dyn_part4.q.out +++ ql/src/test/results/clientpositive/load_dyn_part4.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.nzhang_part4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part4 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/load_dyn_part8.q.out ql/src/test/results/clientpositive/load_dyn_part8.q.out index cb1a757051..02205ee699 100644 --- ql/src/test/results/clientpositive/load_dyn_part8.q.out +++ ql/src/test/results/clientpositive/load_dyn_part8.q.out @@ -364,7 +364,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -396,7 +397,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from srcpart diff --git ql/src/test/results/clientpositive/load_dyn_part9.q.out ql/src/test/results/clientpositive/load_dyn_part9.q.out index 414e784309..1619e0c97f 100644 --- ql/src/test/results/clientpositive/load_dyn_part9.q.out +++ ql/src/test/results/clientpositive/load_dyn_part9.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.nzhang_part9 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part9 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/mapjoin_hook.q.out ql/src/test/results/clientpositive/mapjoin_hook.q.out index a9f9be3a4d..abb2945ec3 100644 --- ql/src/test/results/clientpositive/mapjoin_hook.q.out +++ ql/src/test/results/clientpositive/mapjoin_hook.q.out @@ -16,7 +16,7 @@ PREHOOK: Output: default@dest1 RUN: Stage-6:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value PREHOOK: type: QUERY @@ -26,7 +26,7 @@ PREHOOK: Output: default@dest1 RUN: Stage-6:MAPREDLOCAL RUN: Stage-5:MAPRED RUN: Stage-0:MOVE -RUN: Stage-2:STATS +RUN: Stage-2:COLUMNSTATS PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where (src1.ds = '2008-04-08' or src1.ds = '2008-04-09' )and (src1.hr = '12' or src1.hr = '11') @@ -45,7 +45,7 @@ RUN: Stage-6:CONDITIONAL RUN: Stage-7:MAPREDLOCAL RUN: Stage-1:MAPRED RUN: Stage-0:MOVE -RUN: Stage-2:STATS +RUN: Stage-2:COLUMNSTATS PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value PREHOOK: type: QUERY @@ -63,4 +63,4 @@ RUN: Stage-8:CONDITIONAL RUN: Stage-12:MAPREDLOCAL RUN: Stage-2:MAPRED RUN: Stage-0:MOVE -RUN: Stage-3:STATS +RUN: Stage-3:COLUMNSTATS diff --git ql/src/test/results/clientpositive/mapreduce1.q.out ql/src/test/results/clientpositive/mapreduce1.q.out index 3d0a156557..602ae774af 100644 --- ql/src/test/results/clientpositive/mapreduce1.q.out +++ ql/src/test/results/clientpositive/mapreduce1.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce2.q.out ql/src/test/results/clientpositive/mapreduce2.q.out index 676c387c7a..a401ac0cc1 100644 --- ql/src/test/results/clientpositive/mapreduce2.q.out +++ ql/src/test/results/clientpositive/mapreduce2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce3.q.out ql/src/test/results/clientpositive/mapreduce3.q.out index fc1a402b68..abc3f094c9 100644 --- ql/src/test/results/clientpositive/mapreduce3.q.out +++ ql/src/test/results/clientpositive/mapreduce3.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce4.q.out ql/src/test/results/clientpositive/mapreduce4.q.out index 17fa029ad4..91275a7108 100644 --- ql/src/test/results/clientpositive/mapreduce4.q.out +++ ql/src/test/results/clientpositive/mapreduce4.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce5.q.out ql/src/test/results/clientpositive/mapreduce5.q.out index 21103f88df..bb1e9e42b3 100644 --- ql/src/test/results/clientpositive/mapreduce5.q.out +++ ql/src/test/results/clientpositive/mapreduce5.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce6.q.out ql/src/test/results/clientpositive/mapreduce6.q.out index fe4e631077..1a31ebc1cb 100644 --- ql/src/test/results/clientpositive/mapreduce6.q.out +++ ql/src/test/results/clientpositive/mapreduce6.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce7.q.out ql/src/test/results/clientpositive/mapreduce7.q.out index cc97887fd8..b0c24fa8a8 100644 --- ql/src/test/results/clientpositive/mapreduce7.q.out +++ ql/src/test/results/clientpositive/mapreduce7.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce8.q.out ql/src/test/results/clientpositive/mapreduce8.q.out index b1763c792f..fc866d558d 100644 --- ql/src/test/results/clientpositive/mapreduce8.q.out +++ ql/src/test/results/clientpositive/mapreduce8.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/masking_11.q.out ql/src/test/results/clientpositive/masking_11.q.out index f29c51fdd9..0941b8c2ab 100644 --- ql/src/test/results/clientpositive/masking_11.q.out +++ ql/src/test/results/clientpositive/masking_11.q.out @@ -13,8 +13,10 @@ POSTHOOK: Lineage: masking_test.value SIMPLE [(src)src.FieldSchema(name:value, t PREHOOK: query: analyze table `masking_test` compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test #### A masked pattern was here #### POSTHOOK: query: analyze table `masking_test` compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/merge1.q.out ql/src/test/results/clientpositive/merge1.q.out index 7423d83ac8..bd8cd7bdfa 100644 --- ql/src/test/results/clientpositive/merge1.q.out +++ ql/src/test/results/clientpositive/merge1.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -548,7 +549,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -649,7 +651,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge2.q.out ql/src/test/results/clientpositive/merge2.q.out index bbc55d8b9e..0d781c29ce 100644 --- ql/src/test/results/clientpositive/merge2.q.out +++ ql/src/test/results/clientpositive/merge2.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -548,7 +549,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -649,7 +651,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge3.q.out ql/src/test/results/clientpositive/merge3.q.out index 1eb5f0e03a..5b24a0d29d 100644 --- ql/src/test/results/clientpositive/merge3.q.out +++ ql/src/test/results/clientpositive/merge3.q.out @@ -178,7 +178,8 @@ STAGE PLANS: name: default.merge_src2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -2559,7 +2560,8 @@ STAGE PLANS: name: default.merge_src_part2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -4994,7 +4996,8 @@ STAGE PLANS: name: default.merge_src_part2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/merge4.q.out ql/src/test/results/clientpositive/merge4.q.out index 182c6a887e..de4c593d53 100644 --- ql/src/test/results/clientpositive/merge4.q.out +++ ql/src/test/results/clientpositive/merge4.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.nzhang_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -1185,7 +1186,8 @@ STAGE PLANS: name: default.nzhang_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -2845,7 +2847,8 @@ STAGE PLANS: name: default.nzhang_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition.q.out ql/src/test/results/clientpositive/merge_dynamic_partition.q.out index a777fe0830..990e6dfb84 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table merge_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp where ds='2008-04-08' PREHOOK: type: QUERY @@ -692,7 +693,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -1322,7 +1324,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out index 5a2afb01ac..ccffc89d24 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out @@ -123,7 +123,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out index 055e07abd8..68d7bbe122 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out @@ -183,7 +183,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out index cbeaf42eaf..0da56cd9fc 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out @@ -184,7 +184,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out index 5a562f4456..45237a15b9 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out @@ -160,7 +160,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/metadata_only_queries.q.out ql/src/test/results/clientpositive/metadata_only_queries.q.out index 72a90fbf9d..0fe6684b82 100644 --- ql/src/test/results/clientpositive/metadata_only_queries.q.out +++ ql/src/test/results/clientpositive/metadata_only_queries.q.out @@ -395,40 +395,54 @@ STAGE PLANS: PREHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl +PREHOOK: Output: default@stats_tbl #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl +POSTHOOK: Output: default@stats_tbl #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2011 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2011 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2012 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2012 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### PREHOOK: query: explain select count(*), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si) from stats_tbl diff --git ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out index 6376aa79f8..79d9d27203 100644 --- ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out +++ ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out @@ -126,21 +126,29 @@ PREHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2014 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2014 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### PREHOOK: query: explain select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010 diff --git ql/src/test/results/clientpositive/multi_insert_gby.q.out ql/src/test/results/clientpositive/multi_insert_gby.q.out index cb97e5871f..1633e4e215 100644 --- ql/src/test/results/clientpositive/multi_insert_gby.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -118,7 +119,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 @@ -284,7 +286,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -297,7 +300,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/multi_insert_gby2.q.out ql/src/test/results/clientpositive/multi_insert_gby2.q.out index d1da4e75cd..ceb694052f 100644 --- ql/src/test/results/clientpositive/multi_insert_gby2.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby2.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -108,7 +109,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as value from src order by key) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/multi_insert_gby3.q.out ql/src/test/results/clientpositive/multi_insert_gby3.q.out index 610b15fd1d..d004e644e9 100644 --- ql/src/test/results/clientpositive/multi_insert_gby3.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby3.q.out @@ -111,7 +111,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -124,7 +125,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -215,7 +217,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -228,7 +231,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a INSERT OVERWRITE TABLE e1 @@ -1635,7 +1639,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -1677,7 +1682,8 @@ STAGE PLANS: name: default.e3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -1787,7 +1793,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1800,7 +1807,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -1842,5 +1850,6 @@ STAGE PLANS: name: default.e3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/multi_insert_gby4.q.out ql/src/test/results/clientpositive/multi_insert_gby4.q.out index dd01b74da1..956cd91661 100644 --- ql/src/test/results/clientpositive/multi_insert_gby4.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby4.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -159,7 +160,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -172,7 +174,8 @@ STAGE PLANS: name: default.e3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (SELECT key, value FROM src) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/multi_insert_mixed.q.out ql/src/test/results/clientpositive/multi_insert_mixed.q.out index 9acae2ec54..05c030e0e0 100644 --- ql/src/test/results/clientpositive/multi_insert_mixed.q.out +++ ql/src/test/results/clientpositive/multi_insert_mixed.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -208,7 +209,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -221,7 +223,8 @@ STAGE PLANS: name: default.src_multi3 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select key, count(1) group by key order by key diff --git ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out index af0ef54a74..2254d83e8e 100644 --- ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out +++ ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -97,7 +98,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -241,7 +243,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -254,7 +257,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -448,7 +452,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -461,7 +466,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -605,7 +611,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -618,7 +625,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -825,7 +833,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -838,7 +847,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -988,7 +998,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1001,7 +1012,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -1201,7 +1213,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1214,7 +1227,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1364,7 +1378,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1377,7 +1392,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -1593,7 +1609,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1606,7 +1623,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1798,7 +1816,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1811,7 +1830,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -2053,7 +2073,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2066,7 +2087,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -2258,7 +2280,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2271,7 +2294,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -3374,7 +3398,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3387,7 +3412,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3598,7 +3624,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3611,7 +3638,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3841,7 +3869,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3854,7 +3883,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-12 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce @@ -4153,7 +4183,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4166,7 +4197,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-12 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce diff --git ql/src/test/results/clientpositive/multi_insert_union_src.q.out ql/src/test/results/clientpositive/multi_insert_union_src.q.out index 1ff1db5c12..f7f5d7510d 100644 --- ql/src/test/results/clientpositive/multi_insert_union_src.q.out +++ ql/src/test/results/clientpositive/multi_insert_union_src.q.out @@ -143,7 +143,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -179,7 +180,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src1 where key < 10 union all select * from src2 where key > 100) s insert overwrite table src_multi1 select key, value where key < 150 order by key diff --git ql/src/test/results/clientpositive/multi_insert_with_join2.q.out ql/src/test/results/clientpositive/multi_insert_with_join2.q.out index 225f2c301c..74fcc1f76a 100644 --- ql/src/test/results/clientpositive/multi_insert_with_join2.q.out +++ ql/src/test/results/clientpositive/multi_insert_with_join2.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain @@ -212,7 +213,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a LEFT JOIN T_B b ON a.id = b.id @@ -312,7 +314,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -325,7 +328,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a LEFT JOIN T_B b ON a.id = b.id @@ -425,7 +429,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -438,7 +443,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -544,7 +550,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -557,7 +564,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -663,7 +671,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -676,7 +685,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -794,7 +804,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -807,7 +818,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -925,7 +937,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -938,5 +951,6 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/multigroupby_singlemr.q.out ql/src/test/results/clientpositive/multigroupby_singlemr.q.out index 7af8c4356d..cd030e55ab 100644 --- ql/src/test/results/clientpositive/multigroupby_singlemr.q.out +++ ql/src/test/results/clientpositive/multigroupby_singlemr.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -169,7 +170,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -259,7 +261,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -302,7 +305,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -392,7 +396,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -435,7 +440,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -522,7 +528,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -535,7 +542,8 @@ STAGE PLANS: name: default.dest4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -646,7 +654,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -689,7 +698,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce @@ -732,5 +742,6 @@ STAGE PLANS: name: default.dest1 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/nonmr_fetch.q.out ql/src/test/results/clientpositive/nonmr_fetch.q.out index 560708981d..229a7874ab 100644 --- ql/src/test/results/clientpositive/nonmr_fetch.q.out +++ ql/src/test/results/clientpositive/nonmr_fetch.q.out @@ -1090,7 +1090,8 @@ STAGE PLANS: name: default.srcx Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain analyze table src compute statistics PREHOOK: type: QUERY @@ -1109,7 +1110,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain select * from src join src src2 on src.key=src2.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out index d01461b51f..d6a8da5b15 100644 --- ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out +++ ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 PREHOOK: type: QUERY @@ -155,7 +156,8 @@ STAGE PLANS: name: default.insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 PREHOOK: type: QUERY @@ -249,7 +251,8 @@ STAGE PLANS: name: default.insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/notable_alias1.q.out ql/src/test/results/clientpositive/notable_alias1.q.out index 677545d8d2..ff1778b66b 100644 --- ql/src/test/results/clientpositive/notable_alias1.q.out +++ ql/src/test/results/clientpositive/notable_alias1.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT '1234', key, count(1) WHERE src.key < 100 group by key diff --git ql/src/test/results/clientpositive/notable_alias2.q.out ql/src/test/results/clientpositive/notable_alias2.q.out index 66d0b2a92a..2f8b0e9b27 100644 --- ql/src/test/results/clientpositive/notable_alias2.q.out +++ ql/src/test/results/clientpositive/notable_alias2.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT '1234', src.key, count(1) WHERE key < 100 group by src.key diff --git ql/src/test/results/clientpositive/nullformatCTAS.q.out ql/src/test/results/clientpositive/nullformatCTAS.q.out index cda09658ce..723a4c9ac1 100644 --- ql/src/test/results/clientpositive/nullformatCTAS.q.out +++ ql/src/test/results/clientpositive/nullformatCTAS.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.null_tab3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/optimize_filter_literal.q.out ql/src/test/results/clientpositive/optimize_filter_literal.q.out index 32e2bf6415..ad7e0a418d 100644 --- ql/src/test/results/clientpositive/optimize_filter_literal.q.out +++ ql/src/test/results/clientpositive/optimize_filter_literal.q.out @@ -90,11 +90,15 @@ PREHOOK: query: analyze table tab_part partition (ds='2008-04-08') compute stati PREHOOK: type: QUERY PREHOOK: Input: default@tab_part PREHOOK: Input: default@tab_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part +PREHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab_part partition (ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part +POSTHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS ORCFILE PREHOOK: type: CREATETABLE @@ -122,11 +126,15 @@ PREHOOK: query: analyze table tab partition (ds='2008-04-08') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@tab PREHOOK: Input: default@tab@ds=2008-04-08 +PREHOOK: Output: default@tab +PREHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab partition (ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab POSTHOOK: Input: default@tab@ds=2008-04-08 +POSTHOOK: Output: default@tab +POSTHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from diff --git ql/src/test/results/clientpositive/orc_createas1.q.out ql/src/test/results/clientpositive/orc_createas1.q.out index 506f39d59f..ecfad35286 100644 --- ql/src/test/results/clientpositive/orc_createas1.q.out +++ ql/src/test/results/clientpositive/orc_createas1.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.orc_createas1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -274,7 +275,8 @@ STAGE PLANS: name: default.orc_createas1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge1.q.out ql/src/test/results/clientpositive/orc_merge1.q.out index a83e85bec4..936e93fc06 100644 --- ql/src/test/results/clientpositive/orc_merge1.q.out +++ ql/src/test/results/clientpositive/orc_merge1.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -173,7 +174,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge10.q.out ql/src/test/results/clientpositive/orc_merge10.q.out index 607aaeb6ae..94956e8565 100644 --- ql/src/test/results/clientpositive/orc_merge10.q.out +++ ql/src/test/results/clientpositive/orc_merge10.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -173,7 +174,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -418,7 +421,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/orc_merge2.q.out ql/src/test/results/clientpositive/orc_merge2.q.out index d4c474f9fe..5e8a1f0ccb 100644 --- ql/src/test/results/clientpositive/orc_merge2.q.out +++ ql/src/test/results/clientpositive/orc_merge2.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge3.q.out ql/src/test/results/clientpositive/orc_merge3.q.out index 7bf12c6c28..e527f11795 100644 --- ql/src/test/results/clientpositive/orc_merge3.q.out +++ ql/src/test/results/clientpositive/orc_merge3.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge4.q.out ql/src/test/results/clientpositive/orc_merge4.q.out index 828f204157..c063fc345e 100644 --- ql/src/test/results/clientpositive/orc_merge4.q.out +++ ql/src/test/results/clientpositive/orc_merge4.q.out @@ -122,7 +122,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/orc_merge5.q.out ql/src/test/results/clientpositive/orc_merge5.q.out index e8451869f6..dffec947a4 100644 --- ql/src/test/results/clientpositive/orc_merge5.q.out +++ ql/src/test/results/clientpositive/orc_merge5.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -160,7 +161,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -273,7 +275,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5b concatenate PREHOOK: type: ALTER_TABLE_MERGE diff --git ql/src/test/results/clientpositive/orc_merge6.q.out ql/src/test/results/clientpositive/orc_merge6.q.out index 5ece361bbc..cd84ff5857 100644 --- ql/src/test/results/clientpositive/orc_merge6.q.out +++ ql/src/test/results/clientpositive/orc_merge6.q.out @@ -69,7 +69,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (year="2000",hour=24) select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -208,7 +209,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -408,7 +410,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(year="2000",hour=24) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out index a83e85bec4..936e93fc06 100644 --- ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -173,7 +174,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/orc_merge_incompat1.q.out index fcf1c68f83..29575f9fbb 100644 --- ql/src/test/results/clientpositive/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/orc_merge_incompat1.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/orc_merge_incompat2.q.out index a27041fd9b..158b8e0bec 100644 --- ql/src/test/results/clientpositive/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/orc_merge_incompat2.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 order by userid PREHOOK: type: QUERY @@ -277,7 +278,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/outer_reference_windowed.q.out ql/src/test/results/clientpositive/outer_reference_windowed.q.out index 003a04cddd..129d39db8a 100644 --- ql/src/test/results/clientpositive/outer_reference_windowed.q.out +++ ql/src/test/results/clientpositive/outer_reference_windowed.q.out @@ -93,26 +93,32 @@ POSTHOOK: Lineage: e011_03.c2 SIMPLE [(e011_01)e011_01.FieldSchema(name:c2, type PREHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_01 +PREHOOK: Output: default@e011_01 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_01 +POSTHOOK: Output: default@e011_01 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_02 +PREHOOK: Output: default@e011_02 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_02 +POSTHOOK: Output: default@e011_02 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_03 +PREHOOK: Output: default@e011_03 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_03 +POSTHOOK: Output: default@e011_03 #### A masked pattern was here #### PREHOOK: query: explain select sum(sum(c1)) over() from e011_01 PREHOOK: type: QUERY @@ -129,11 +135,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)) outputColumnNames: c1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) mode: hash @@ -239,22 +245,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: c1, c2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) keys: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Group By Operator @@ -262,7 +268,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -278,13 +284,13 @@ STAGE PLANS: key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col1 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -304,14 +310,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -365,19 +371,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -402,13 +408,13 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) keys: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -424,7 +430,7 @@ STAGE PLANS: key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Group By Operator @@ -432,7 +438,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -448,13 +454,13 @@ STAGE PLANS: key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col1 (type: decimal(15,2)) - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -474,14 +480,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -539,19 +545,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE TableScan alias: e011_03 Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE @@ -576,13 +582,13 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) keys: _col1 (type: decimal(15,2)), _col2 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -598,7 +604,7 @@ STAGE PLANS: key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Group By Operator @@ -606,7 +612,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -622,13 +628,13 @@ STAGE PLANS: key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col1 (type: decimal(15,2)) - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -648,14 +654,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -712,19 +718,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -750,13 +756,13 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: corr(_col0, _col2) keys: _col1 (type: decimal(15,2)), _col3 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -772,7 +778,7 @@ STAGE PLANS: key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -780,11 +786,11 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)), _col2 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -804,14 +810,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumDouble window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/parallel.q.out ql/src/test/results/clientpositive/parallel.q.out index 459105e09a..31b1a0dd3d 100644 --- ql/src/test/results/clientpositive/parallel.q.out +++ ql/src/test/results/clientpositive/parallel.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -129,7 +130,8 @@ STAGE PLANS: name: default.src_b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/parallel_colstats.q.out ql/src/test/results/clientpositive/parallel_colstats.q.out index 83fc14a72f..12ad5ee874 100644 --- ql/src/test/results/clientpositive/parallel_colstats.q.out +++ ql/src/test/results/clientpositive/parallel_colstats.q.out @@ -28,12 +28,10 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-4, Stage-5, Stage-6, Stage-7 - Stage-9 depends on stages: Stage-4, Stage-5, Stage-6, Stage-7 + Stage-4 depends on stages: Stage-0, Stage-5, Stage-7 Stage-5 depends on stages: Stage-3 + Stage-6 depends on stages: Stage-1, Stage-5, Stage-7 Stage-1 depends on stages: Stage-3 - Stage-6 depends on stages: Stage-1 Stage-7 depends on stages: Stage-3 STAGE PLANS: @@ -150,22 +148,13 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string Table: default.src_a - Stage: Stage-9 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.src_b - Stage: Stage-5 Map Reduce Map Operator Tree: @@ -188,6 +177,14 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-6 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.src_b + Stage: Stage-1 Move Operator tables: @@ -198,9 +195,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_b - Stage: Stage-6 - Stats-Aggr Operator - Stage: Stage-7 Map Reduce Map Operator Tree: diff --git ql/src/test/results/clientpositive/parallel_join1.q.out ql/src/test/results/clientpositive/parallel_join1.q.out index 8843661176..0450e1586d 100644 --- ql/src/test/results/clientpositive/parallel_join1.q.out +++ ql/src/test/results/clientpositive/parallel_join1.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/parallel_orderby.q.out ql/src/test/results/clientpositive/parallel_orderby.q.out index 8249a7baad..9a30336194 100644 --- ql/src/test/results/clientpositive/parallel_orderby.q.out +++ ql/src/test/results/clientpositive/parallel_orderby.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.total_ordered Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table total_ordered as select * from src5 order by key, value PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/partial_column_stats.q.out ql/src/test/results/clientpositive/partial_column_stats.q.out index e8b1a99fe2..04a00cdace 100644 --- ql/src/test/results/clientpositive/partial_column_stats.q.out +++ ql/src/test/results/clientpositive/partial_column_stats.q.out @@ -49,7 +49,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string @@ -58,10 +59,12 @@ STAGE PLANS: PREHOOK: query: analyze table t1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 #### A masked pattern was here #### POSTHOOK: query: analyze table t1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 #### A masked pattern was here #### PREHOOK: query: desc formatted t1 value PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/partition_coltype_literals.q.out ql/src/test/results/clientpositive/partition_coltype_literals.q.out index 51eda726d6..6759983c27 100644 --- ql/src/test/results/clientpositive/partition_coltype_literals.q.out +++ ql/src/test/results/clientpositive/partition_coltype_literals.q.out @@ -290,11 +290,15 @@ PREHOOK: query: analyze table partcoltypenum partition (tint=110Y, sint=22000S, PREHOOK: type: QUERY PREHOOK: Input: default@partcoltypenum PREHOOK: Input: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 +PREHOOK: Output: default@partcoltypenum +PREHOOK: Output: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 #### A masked pattern was here #### POSTHOOK: query: analyze table partcoltypenum partition (tint=110Y, sint=22000S, bint=330000000000L) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcoltypenum POSTHOOK: Input: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 +POSTHOOK: Output: default@partcoltypenum +POSTHOOK: Output: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 #### A masked pattern was here #### PREHOOK: query: describe formatted partcoltypenum partition (tint=110Y, sint=22000S, bint=330000000000L) key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/pcr.q.out ql/src/test/results/clientpositive/pcr.q.out index 5d0d170245..0a200d63af 100644 --- ql/src/test/results/clientpositive/pcr.q.out +++ ql/src/test/results/clientpositive/pcr.q.out @@ -3649,7 +3649,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -3870,7 +3871,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 @@ -4279,7 +4281,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -4500,7 +4503,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 diff --git ql/src/test/results/clientpositive/pcs.q.out ql/src/test/results/clientpositive/pcs.q.out index af5d11ad44..6b2ad02cb5 100644 --- ql/src/test/results/clientpositive/pcs.q.out +++ ql/src/test/results/clientpositive/pcs.q.out @@ -70,6 +70,10 @@ PREHOOK: Input: default@pcs_t1 PREHOOK: Input: default@pcs_t1@ds=2000-04-08 PREHOOK: Input: default@pcs_t1@ds=2000-04-09 PREHOOK: Input: default@pcs_t1@ds=2000-04-10 +PREHOOK: Output: default@pcs_t1 +PREHOOK: Output: default@pcs_t1@ds=2000-04-08 +PREHOOK: Output: default@pcs_t1@ds=2000-04-09 +PREHOOK: Output: default@pcs_t1@ds=2000-04-10 #### A masked pattern was here #### POSTHOOK: query: analyze table pcs_t1 partition(ds) compute statistics for columns POSTHOOK: type: QUERY @@ -77,6 +81,10 @@ POSTHOOK: Input: default@pcs_t1 POSTHOOK: Input: default@pcs_t1@ds=2000-04-08 POSTHOOK: Input: default@pcs_t1@ds=2000-04-09 POSTHOOK: Input: default@pcs_t1@ds=2000-04-10 +POSTHOOK: Output: default@pcs_t1 +POSTHOOK: Output: default@pcs_t1@ds=2000-04-08 +POSTHOOK: Output: default@pcs_t1@ds=2000-04-09 +POSTHOOK: Output: default@pcs_t1@ds=2000-04-10 #### A masked pattern was here #### PREHOOK: query: explain extended select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/ppd_constant_expr.q.out ql/src/test/results/clientpositive/ppd_constant_expr.q.out index cbe76549f7..ca35c4c393 100644 --- ql/src/test/results/clientpositive/ppd_constant_expr.q.out +++ ql/src/test/results/clientpositive/ppd_constant_expr.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.ppd_constant_expr Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -200,7 +201,8 @@ STAGE PLANS: name: default.ppd_constant_expr Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/ppd_multi_insert.q.out ql/src/test/results/clientpositive/ppd_multi_insert.q.out index 7e501c71c0..8583794e17 100644 --- ql/src/test/results/clientpositive/ppd_multi_insert.q.out +++ ql/src/test/results/clientpositive/ppd_multi_insert.q.out @@ -161,7 +161,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -174,7 +175,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -190,7 +192,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1426,7 +1429,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1439,7 +1443,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1455,7 +1460,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator diff --git ql/src/test/results/clientpositive/quote1.q.out ql/src/test/results/clientpositive/quote1.q.out index f8592c4e00..740b3c9044 100644 --- ql/src/test/results/clientpositive/quote1.q.out +++ ql/src/test/results/clientpositive/quote1.q.out @@ -69,7 +69,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/quotedid_stats.q.out ql/src/test/results/clientpositive/quotedid_stats.q.out index 40e23bf773..b85eb26568 100644 --- ql/src/test/results/clientpositive/quotedid_stats.q.out +++ ql/src/test/results/clientpositive/quotedid_stats.q.out @@ -44,10 +44,12 @@ Storage Desc Params: PREHOOK: query: analyze table t4 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@t4 +PREHOOK: Output: default@t4 #### A masked pattern was here #### POSTHOOK: query: analyze table t4 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@t4 +POSTHOOK: Output: default@t4 #### A masked pattern was here #### PREHOOK: query: describe formatted t4 PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/rand_partitionpruner2.q.out ql/src/test/results/clientpositive/rand_partitionpruner2.q.out index 9b29136ba1..1482fde260 100644 --- ql/src/test/results/clientpositive/rand_partitionpruner2.q.out +++ ql/src/test/results/clientpositive/rand_partitionpruner2.q.out @@ -213,7 +213,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/rcfile_null_value.q.out ql/src/test/results/clientpositive/rcfile_null_value.q.out index 12b714088a..799adfe52d 100644 --- ql/src/test/results/clientpositive/rcfile_null_value.q.out +++ ql/src/test/results/clientpositive/rcfile_null_value.q.out @@ -161,7 +161,8 @@ STAGE PLANS: name: default.dest1_rc Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/remove_exprs_stats.q.out ql/src/test/results/clientpositive/remove_exprs_stats.q.out index 4600e71f09..28caeb833d 100644 --- ql/src/test/results/clientpositive/remove_exprs_stats.q.out +++ ql/src/test/results/clientpositive/remove_exprs_stats.q.out @@ -55,10 +55,12 @@ POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select * from loc_orc where locid < 30 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/rename_external_partition_location.q.out ql/src/test/results/clientpositive/rename_external_partition_location.q.out index 9665bce88a..5fb71a49f5 100644 --- ql/src/test/results/clientpositive/rename_external_partition_location.q.out +++ ql/src/test/results/clientpositive/rename_external_partition_location.q.out @@ -69,11 +69,15 @@ PREHOOK: query: ANALYZE TABLE ex_table PARTITION (part='part1') COMPUTE STATISTI PREHOOK: type: QUERY PREHOOK: Input: default@ex_table PREHOOK: Input: default@ex_table@part=part1 +PREHOOK: Output: default@ex_table +PREHOOK: Output: default@ex_table@part=part1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE ex_table PARTITION (part='part1') COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@ex_table POSTHOOK: Input: default@ex_table@part=part1 +POSTHOOK: Output: default@ex_table +POSTHOOK: Output: default@ex_table@part=part1 #### A masked pattern was here #### PREHOOK: query: DESCRIBE FORMATTED ex_table PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out index d135485991..a62575f0e0 100644 --- ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out +++ ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out @@ -47,10 +47,12 @@ PREHOOK: query: analyze table testtable1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable1 POSTHOOK: query: analyze table testtable1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable1 PREHOOK: query: describe formatted statsdb1.testtable1 col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable1 @@ -193,10 +195,12 @@ PREHOOK: query: analyze table testtable1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable1 POSTHOOK: query: analyze table testtable1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable1 PREHOOK: query: describe formatted statsdb1.testtable1 col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable1 diff --git ql/src/test/results/clientpositive/sample1.q.out ql/src/test/results/clientpositive/sample1.q.out index dec9b233b3..72c32b4506 100644 --- ql/src/test/results/clientpositive/sample1.q.out +++ ql/src/test/results/clientpositive/sample1.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample2.q.out ql/src/test/results/clientpositive/sample2.q.out index f54c57363a..e09fd50225 100644 --- ql/src/test/results/clientpositive/sample2.q.out +++ ql/src/test/results/clientpositive/sample2.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample4.q.out ql/src/test/results/clientpositive/sample4.q.out index 675fda9ef4..46a1c6f425 100644 --- ql/src/test/results/clientpositive/sample4.q.out +++ ql/src/test/results/clientpositive/sample4.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample5.q.out ql/src/test/results/clientpositive/sample5.q.out index 583784e9b7..15e89d0d98 100644 --- ql/src/test/results/clientpositive/sample5.q.out +++ ql/src/test/results/clientpositive/sample5.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample6.q.out ql/src/test/results/clientpositive/sample6.q.out index 36e6906785..c568bd2742 100644 --- ql/src/test/results/clientpositive/sample6.q.out +++ ql/src/test/results/clientpositive/sample6.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample7.q.out ql/src/test/results/clientpositive/sample7.q.out index f0d9088174..6ad8ced7d1 100644 --- ql/src/test/results/clientpositive/sample7.q.out +++ ql/src/test/results/clientpositive/sample7.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/skewjoin.q.out ql/src/test/results/clientpositive/skewjoin.q.out index cd7d6fa053..e5d8db3cc7 100644 --- ql/src/test/results/clientpositive/skewjoin.q.out +++ ql/src/test/results/clientpositive/skewjoin.q.out @@ -198,7 +198,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/skewjoin_noskew.q.out ql/src/test/results/clientpositive/skewjoin_noskew.q.out index 243a840406..d39acaf10a 100644 --- ql/src/test/results/clientpositive/skewjoin_noskew.q.out +++ ql/src/test/results/clientpositive/skewjoin_noskew.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.noskew Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table noskew as select a.* from src a join src b on a.key=b.key order by a.key limit 30 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out index 09aaaf684a..a5c0df9aa7 100644 --- ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out +++ ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out @@ -186,7 +186,8 @@ STAGE PLANS: name: default.result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TABLE result AS SELECT a.* FROM skewtable a JOIN nonskewtable b ON a.key=b.key PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/smb_mapjoin9.q.out ql/src/test/results/clientpositive/smb_mapjoin9.q.out index 6916ce8f0a..433344f302 100644 --- ql/src/test/results/clientpositive/smb_mapjoin9.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin9.q.out @@ -280,7 +280,8 @@ STAGE PLANS: name: default.smb_mapjoin9_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/smb_mapjoin_11.q.out index b53e6704cc..b76e7a8221 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_11.q.out @@ -202,7 +202,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' diff --git ql/src/test/results/clientpositive/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/smb_mapjoin_12.q.out index 9af4683451..6def3de813 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_12.q.out @@ -204,7 +204,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' @@ -409,7 +410,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') diff --git ql/src/test/results/clientpositive/smb_mapjoin_21.q.out ql/src/test/results/clientpositive/smb_mapjoin_21.q.out index c0fdfd38d2..8b568eb258 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_21.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_21.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -153,7 +154,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -229,7 +231,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -306,7 +309,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -383,7 +387,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -459,5 +464,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/smb_mapjoin_22.q.out ql/src/test/results/clientpositive/smb_mapjoin_22.q.out index 36e879236f..256a3c26e0 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_22.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_22.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 @@ -212,7 +213,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 diff --git ql/src/test/results/clientpositive/smb_mapjoin_7.q.out ql/src/test/results/clientpositive/smb_mapjoin_7.q.out index 82f5804eea..a2a8660c3b 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_7.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_7.q.out @@ -669,7 +669,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out index 02d1605f40..118a48eb57 100644 --- ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out +++ ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out @@ -97,26 +97,32 @@ POSTHOOK: Output: default@loc PREHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid PREHOOK: type: QUERY PREHOOK: Input: default@emp +PREHOOK: Output: default@emp #### A masked pattern was here #### POSTHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp +POSTHOOK: Output: default@emp #### A masked pattern was here #### PREHOOK: query: analyze table dept compute statistics for columns deptname,deptid PREHOOK: type: QUERY PREHOOK: Input: default@dept +PREHOOK: Output: default@dept #### A masked pattern was here #### POSTHOOK: query: analyze table dept compute statistics for columns deptname,deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@dept +POSTHOOK: Output: default@dept #### A masked pattern was here #### PREHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc +PREHOOK: Output: default@loc #### A masked pattern was here #### POSTHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc +POSTHOOK: Output: default@loc #### A masked pattern was here #### PREHOOK: query: explain select * from emp e join dept d on (e.deptid = d.deptid) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/auto_join1.q.out ql/src/test/results/clientpositive/spark/auto_join1.q.out index d9cd7700cc..49795dc71c 100644 --- ql/src/test/results/clientpositive/spark/auto_join1.q.out +++ ql/src/test/results/clientpositive/spark/auto_join1.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/auto_join14.q.out ql/src/test/results/clientpositive/spark/auto_join14.q.out index 82deefea73..356cc85703 100644 --- ql/src/test/results/clientpositive/spark/auto_join14.q.out +++ ql/src/test/results/clientpositive/spark/auto_join14.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/spark/auto_join17.q.out ql/src/test/results/clientpositive/spark/auto_join17.q.out index 6d63fa68a3..b3e01f2a08 100644 --- ql/src/test/results/clientpositive/spark/auto_join17.q.out +++ ql/src/test/results/clientpositive/spark/auto_join17.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* diff --git ql/src/test/results/clientpositive/spark/auto_join19.q.out ql/src/test/results/clientpositive/spark/auto_join19.q.out index 88ef3f1981..75426c15c1 100644 --- ql/src/test/results/clientpositive/spark/auto_join19.q.out +++ ql/src/test/results/clientpositive/spark/auto_join19.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/auto_join2.q.out ql/src/test/results/clientpositive/spark/auto_join2.q.out index e32abba2f4..126acada5b 100644 --- ql/src/test/results/clientpositive/spark/auto_join2.q.out +++ ql/src/test/results/clientpositive/spark/auto_join2.q.out @@ -124,7 +124,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/auto_join26.q.out ql/src/test/results/clientpositive/spark/auto_join26.q.out index bfb3564cf8..33e160993f 100644 --- ql/src/test/results/clientpositive/spark/auto_join26.q.out +++ ql/src/test/results/clientpositive/spark/auto_join26.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/spark/auto_join3.q.out ql/src/test/results/clientpositive/spark/auto_join3.q.out index a17cc1a0a1..11f909d4b8 100644 --- ql/src/test/results/clientpositive/spark/auto_join3.q.out +++ ql/src/test/results/clientpositive/spark/auto_join3.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/auto_join4.q.out ql/src/test/results/clientpositive/spark/auto_join4.q.out index ca71d9f2ed..bdd4f97129 100644 --- ql/src/test/results/clientpositive/spark/auto_join4.q.out +++ ql/src/test/results/clientpositive/spark/auto_join4.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join5.q.out ql/src/test/results/clientpositive/spark/auto_join5.q.out index 2b11a96767..b01bab6cab 100644 --- ql/src/test/results/clientpositive/spark/auto_join5.q.out +++ ql/src/test/results/clientpositive/spark/auto_join5.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join6.q.out ql/src/test/results/clientpositive/spark/auto_join6.q.out index 4f1e6316b8..329676732c 100644 --- ql/src/test/results/clientpositive/spark/auto_join6.q.out +++ ql/src/test/results/clientpositive/spark/auto_join6.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join7.q.out ql/src/test/results/clientpositive/spark/auto_join7.q.out index 0e6e2f5b73..a6982a82f6 100644 --- ql/src/test/results/clientpositive/spark/auto_join7.q.out +++ ql/src/test/results/clientpositive/spark/auto_join7.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join8.q.out ql/src/test/results/clientpositive/spark/auto_join8.q.out index c0795366f5..40e439f3d4 100644 --- ql/src/test/results/clientpositive/spark/auto_join8.q.out +++ ql/src/test/results/clientpositive/spark/auto_join8.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join9.q.out ql/src/test/results/clientpositive/spark/auto_join9.q.out index b2ed51c108..671eafe0b3 100644 --- ql/src/test/results/clientpositive/spark/auto_join9.q.out +++ ql/src/test/results/clientpositive/spark/auto_join9.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12' diff --git ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out index c26949f13c..9308d476e3 100644 --- ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out @@ -1391,7 +1391,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1404,7 +1405,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -1611,7 +1613,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1624,7 +1627,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out index cd4b83a2a1..ded68d5e72 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out @@ -134,7 +134,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -147,7 +148,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -319,7 +321,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -332,7 +335,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -532,7 +536,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -545,7 +550,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 diff --git ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out index eb2a636167..4b8112393c 100644 --- ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out +++ ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out @@ -25,10 +25,12 @@ POSTHOOK: Output: default@dec PREHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value PREHOOK: type: QUERY PREHOOK: Input: default@dec +PREHOOK: Output: default@dec #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value POSTHOOK: type: QUERY POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec #### A masked pattern was here #### PREHOOK: query: DESC FORMATTED `dec` value PREHOOK: type: DESCTABLE @@ -39,7 +41,7 @@ POSTHOOK: Input: default@dec # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector value decimal(8,4) -12.25 234.79 0 10 from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"value\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/spark/bucket2.q.out ql/src/test/results/clientpositive/spark/bucket2.q.out index 90c9e5469d..32f12582af 100644 --- ql/src/test/results/clientpositive/spark/bucket2.q.out +++ ql/src/test/results/clientpositive/spark/bucket2.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/spark/bucket3.q.out ql/src/test/results/clientpositive/spark/bucket3.q.out index 078460f9b9..b88e73fab8 100644 --- ql/src/test/results/clientpositive/spark/bucket3.q.out +++ ql/src/test/results/clientpositive/spark/bucket3.q.out @@ -163,7 +163,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') diff --git ql/src/test/results/clientpositive/spark/bucket4.q.out ql/src/test/results/clientpositive/spark/bucket4.q.out index 13e21b6610..63683fc8bd 100644 --- ql/src/test/results/clientpositive/spark/bucket4.q.out +++ ql/src/test/results/clientpositive/spark/bucket4.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.bucket4_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket4_1 diff --git ql/src/test/results/clientpositive/spark/bucket5.q.out ql/src/test/results/clientpositive/spark/bucket5.q.out index dd24db8e3e..39a3145f67 100644 --- ql/src/test/results/clientpositive/spark/bucket5.q.out +++ ql/src/test/results/clientpositive/spark/bucket5.q.out @@ -291,7 +291,8 @@ STAGE PLANS: name: default.bucketed_table Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -323,7 +324,8 @@ STAGE PLANS: name: default.unbucketed_table Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM src diff --git ql/src/test/results/clientpositive/spark/bucket6.q.out ql/src/test/results/clientpositive/spark/bucket6.q.out index d5d53d303d..b5d33471eb 100644 --- ql/src/test/results/clientpositive/spark/bucket6.q.out +++ ql/src/test/results/clientpositive/spark/bucket6.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.src_bucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table src_bucket select key,value from srcpart PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out index 194f16ed9c..71063f9316 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out @@ -363,7 +363,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -659,7 +660,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out index bb66d1ec47..04a3a491a0 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out @@ -347,7 +347,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -654,7 +655,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out index 10678a4c26..6ee414c5b0 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out @@ -347,7 +347,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -643,7 +644,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out index 724f9829db..94657001d9 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out @@ -613,7 +613,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -968,7 +969,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out index 49d7cc9928..e801a80f5b 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out @@ -341,7 +341,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -702,7 +703,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -1129,7 +1131,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out index 1db5b2c142..d10a30cc3a 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out @@ -365,7 +365,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -726,7 +727,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out index 7fb61f0645..b2e8262b4a 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out @@ -361,7 +361,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -706,7 +707,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out index a9415135fb..410a8c88a0 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out @@ -460,7 +460,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -866,7 +867,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out index 3e74c217e7..03ca99834d 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out @@ -301,6 +301,7 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out index a3acd5dd74..74027be8c4 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out @@ -370,6 +370,7 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out index 81a064b2b7..814553d81a 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out @@ -186,7 +186,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -362,7 +363,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -562,7 +564,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -768,7 +771,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -956,7 +960,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.v1, b.v2) @@ -1144,7 +1149,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key+a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out index 4245aa1d99..faada6fff0 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key, concat(a.value, b.value) @@ -345,7 +346,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.value diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out index 5e4e5ef8ad..8282a5a33f 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key2, concat(a.value, b.value) @@ -347,7 +348,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq1.key, subq1.key2, subq1.value from @@ -530,7 +532,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -648,7 +651,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -772,7 +776,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.key, subq2.key2, subq2.value from @@ -973,7 +978,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.k2, subq2.k1, subq2.value from @@ -1184,5 +1190,6 @@ STAGE PLANS: name: default.test_table4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out index 7c23da7cc2..f0dd14506f 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -344,7 +345,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -530,7 +532,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out index 8263b6f681..2198dc2fa2 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out @@ -162,7 +162,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, b.key, concat(a.value, b.value) @@ -339,7 +340,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT b.key, a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/spark/ctas.q.out ql/src/test/results/clientpositive/spark/ctas.q.out index 49699308e2..222e44be6a 100644 --- ql/src/test/results/clientpositive/spark/ctas.q.out +++ ql/src/test/results/clientpositive/spark/ctas.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.nzhang_CTAS1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -241,7 +242,8 @@ STAGE PLANS: name: default.nzhang_ctas2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas2 as select * from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -388,7 +390,8 @@ STAGE PLANS: name: default.nzhang_ctas3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -600,7 +603,8 @@ STAGE PLANS: name: default.nzhang_ctas4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -751,7 +755,8 @@ STAGE PLANS: name: default.nzhang_ctas5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out index 8693026c71..86d6e1a874 100644 --- ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out +++ ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out index 1ed388f0c7..3d22cadbee 100644 --- ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out +++ ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out @@ -239,7 +239,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -252,7 +253,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY @@ -453,7 +455,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: SELECT * FROM tmptable x SORT BY x.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1.q.out ql/src/test/results/clientpositive/spark/groupby1.q.out index 42ce2430d3..68cb6d7176 100644 --- ql/src/test/results/clientpositive/spark/groupby1.q.out +++ ql/src/test/results/clientpositive/spark/groupby1.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby10.q.out ql/src/test/results/clientpositive/spark/groupby10.q.out index b572995b32..83bb55d747 100644 --- ql/src/test/results/clientpositive/spark/groupby10.q.out +++ ql/src/test/results/clientpositive/spark/groupby10.q.out @@ -151,7 +151,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -164,7 +165,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -363,7 +365,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -376,7 +379,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -544,7 +548,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -557,7 +562,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, sum(distinct substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key diff --git ql/src/test/results/clientpositive/spark/groupby11.q.out ql/src/test/results/clientpositive/spark/groupby11.q.out index a0f99c4cfa..94ab3e0c44 100644 --- ql/src/test/results/clientpositive/spark/groupby11.q.out +++ ql/src/test/results/clientpositive/spark/groupby11.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -170,7 +171,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 partition(ds='111') diff --git ql/src/test/results/clientpositive/spark/groupby1_map.q.out ql/src/test/results/clientpositive/spark/groupby1_map.q.out index b414aa62a3..775ca8a6b8 100644 --- ql/src/test/results/clientpositive/spark/groupby1_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_map.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out index a01cee1d49..20e9c6566e 100644 --- ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out index f7b7f7a185..8c3f571830 100644 --- ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out index 1b7e53b3eb..ea607cdf82 100644 --- ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby2.q.out ql/src/test/results/clientpositive/spark/groupby2.q.out index a5cd0e6066..966c273235 100644 --- ql/src/test/results/clientpositive/spark/groupby2.q.out +++ ql/src/test/results/clientpositive/spark/groupby2.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_map.q.out ql/src/test/results/clientpositive/spark/groupby2_map.q.out index d2b69af851..c716bbd579 100644 --- ql/src/test/results/clientpositive/spark/groupby2_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_map.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out index 4ad056887d..faf69b68f6 100644 --- ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) @@ -187,7 +188,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.key,1,1)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out index f4a567ea9b..e2974249aa 100644 --- ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out index 8ecf769b43..83e794a9c1 100644 --- ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out index 3ede0fc755..171932ad1b 100644 --- ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby3.q.out ql/src/test/results/clientpositive/spark/groupby3.q.out index a34e89e1ef..c314763db2 100644 --- ql/src/test/results/clientpositive/spark/groupby3.q.out +++ ql/src/test/results/clientpositive/spark/groupby3.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_map.q.out ql/src/test/results/clientpositive/spark/groupby3_map.q.out index 71f8dc0191..9caf47b676 100644 --- ql/src/test/results/clientpositive/spark/groupby3_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_map.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out index 47ef5cb6ae..678276115f 100644 --- ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out index 7cfca81d71..2f101601fe 100644 --- ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out index b2993a6e85..d07eea5fe6 100644 --- ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out index d152a07c77..8bcb912991 100644 --- ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby4.q.out ql/src/test/results/clientpositive/spark/groupby4.q.out index 3ad01d0d70..1146c8404d 100644 --- ql/src/test/results/clientpositive/spark/groupby4.q.out +++ ql/src/test/results/clientpositive/spark/groupby4.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby4_map.q.out ql/src/test/results/clientpositive/spark/groupby4_map.q.out index 7cb360033e..96f34883ac 100644 --- ql/src/test/results/clientpositive/spark/groupby4_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby4_map.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out index ef287ad05a..3bf8d9985e 100644 --- ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out index 04f58fa671..4af92e8b94 100644 --- ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby5.q.out ql/src/test/results/clientpositive/spark/groupby5.q.out index d292f747e6..8dfed8be31 100644 --- ql/src/test/results/clientpositive/spark/groupby5.q.out +++ ql/src/test/results/clientpositive/spark/groupby5.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/spark/groupby5_map.q.out ql/src/test/results/clientpositive/spark/groupby5_map.q.out index add30941b2..96cf6b7fd6 100644 --- ql/src/test/results/clientpositive/spark/groupby5_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby5_map.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out index 924ef5dbc7..11a65580ab 100644 --- ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out index 300ccb64c3..16ac57bbfa 100644 --- ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/spark/groupby6.q.out ql/src/test/results/clientpositive/spark/groupby6.q.out index 4f406d7d07..04e6127413 100644 --- ql/src/test/results/clientpositive/spark/groupby6.q.out +++ ql/src/test/results/clientpositive/spark/groupby6.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby6_map.q.out ql/src/test/results/clientpositive/spark/groupby6_map.q.out index 03f68c63b2..509014a2b7 100644 --- ql/src/test/results/clientpositive/spark/groupby6_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby6_map.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out index 606b5d5272..e2260eda2b 100644 --- ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out index eb72f011b2..f95551942c 100644 --- ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby7_map.q.out ql/src/test/results/clientpositive/spark/groupby7_map.q.out index 31daab8acc..9260e7d408 100644 --- ql/src/test/results/clientpositive/spark/groupby7_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_map.q.out @@ -112,7 +112,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -125,7 +126,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out index 625a7374c9..c628b3932b 100644 --- ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -118,7 +119,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out index 4fbfd30ec6..e55a79d40b 100644 --- ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -140,7 +141,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out index a26247af8d..9e05b99ac9 100644 --- ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out @@ -106,7 +106,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -119,7 +120,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out index 2dce3016b3..9a9f7b28aa 100644 --- ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out @@ -125,7 +125,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -138,7 +139,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key ORDER BY SRC.key limit 10 diff --git ql/src/test/results/clientpositive/spark/groupby8.q.out ql/src/test/results/clientpositive/spark/groupby8.q.out index f7f3279e6b..c66a53dda7 100644 --- ql/src/test/results/clientpositive/spark/groupby8.q.out +++ ql/src/test/results/clientpositive/spark/groupby8.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -133,7 +134,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -893,7 +895,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -906,7 +909,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby8_map.q.out ql/src/test/results/clientpositive/spark/groupby8_map.q.out index 288ca3f3b2..25c07d0ba6 100644 --- ql/src/test/results/clientpositive/spark/groupby8_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby8_map.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -117,7 +118,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out index 9e76fd57aa..1da8890949 100644 --- ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -139,7 +140,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out index 288ca3f3b2..25c07d0ba6 100644 --- ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -117,7 +118,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby9.q.out ql/src/test/results/clientpositive/spark/groupby9.q.out index d59d8cf706..c016e4b6e2 100644 --- ql/src/test/results/clientpositive/spark/groupby9.q.out +++ ql/src/test/results/clientpositive/spark/groupby9.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -144,7 +145,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -916,7 +918,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -929,7 +932,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -1701,7 +1705,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1714,7 +1719,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -2488,7 +2494,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2501,7 +2508,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -3273,7 +3281,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3286,7 +3295,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out index 169ee04403..3b5ec42472 100644 --- ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out +++ ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out @@ -175,7 +175,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -188,7 +189,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -201,7 +203,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) diff --git ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out index 7a730d230d..71e74a92ae 100644 --- ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -172,7 +173,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10 diff --git ql/src/test/results/clientpositive/spark/groupby_cube1.q.out ql/src/test/results/clientpositive/spark/groupby_cube1.q.out index 52c87efaf0..7629fe6620 100644 --- ql/src/test/results/clientpositive/spark/groupby_cube1.q.out +++ ql/src/test/results/clientpositive/spark/groupby_cube1.q.out @@ -704,7 +704,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -717,7 +718,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with cube diff --git ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out index f3a95bdde5..d1fcd48084 100644 --- ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out +++ ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out @@ -234,7 +234,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out index 666f355a4c..30e5daa437 100644 --- ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out @@ -234,7 +234,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out index bf4132a974..f88ff7ae9f 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -144,7 +145,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table dest1 select key, count(distinct value) group by key diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out index c16df1b667..89db8ce177 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -170,7 +171,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -183,7 +185,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) @@ -468,7 +471,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -481,7 +485,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -494,7 +499,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -507,7 +513,8 @@ STAGE PLANS: name: default.dest_h2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Move Operator @@ -520,7 +527,8 @@ STAGE PLANS: name: default.dest_h3 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out index 37deb9336f..a90082f341 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -123,7 +124,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out index b6127475ea..669b6333c4 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out @@ -122,7 +122,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -135,7 +136,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -294,7 +296,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -307,7 +310,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -466,7 +470,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -479,7 +484,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -638,7 +644,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -651,7 +658,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 diff --git ql/src/test/results/clientpositive/spark/groupby_position.q.out ql/src/test/results/clientpositive/spark/groupby_position.q.out index ae27cc1a87..65591d73f3 100644 --- ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -142,7 +143,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 @@ -320,7 +322,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -333,7 +336,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 diff --git ql/src/test/results/clientpositive/spark/groupby_ppr.q.out ql/src/test/results/clientpositive/spark/groupby_ppr.q.out index e45b5c2030..2bbffaf77e 100644 --- ql/src/test/results/clientpositive/spark/groupby_ppr.q.out +++ ql/src/test/results/clientpositive/spark/groupby_ppr.q.out @@ -227,7 +227,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out index c3c97f5a10..14c287f13d 100644 --- ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out @@ -227,7 +227,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src @@ -494,7 +495,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out index 68670ab2fc..29b111fc51 100644 --- ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out +++ ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out @@ -537,7 +537,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -550,7 +551,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with rollup diff --git ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 8292e3a6b4..5ceb9e9dd9 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -189,7 +189,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -400,7 +401,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -585,7 +587,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -768,7 +771,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -959,7 +963,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1171,7 +1176,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -1377,7 +1383,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1595,7 +1602,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -1897,7 +1905,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2232,7 +2241,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2538,7 +2548,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3014,7 +3025,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3197,7 +3209,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3391,7 +3404,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl5 @@ -3584,7 +3598,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3784,7 +3799,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3941,7 +3957,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3954,7 +3971,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -4105,7 +4123,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4118,7 +4137,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 42847269f3..4b11eb48bc 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -189,7 +189,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -419,7 +420,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -604,7 +606,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -787,7 +790,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -978,7 +982,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1209,7 +1214,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -1434,7 +1440,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1671,7 +1678,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -1973,7 +1981,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2327,7 +2336,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2633,7 +2643,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3147,7 +3158,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3330,7 +3342,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3524,7 +3537,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl5 @@ -3717,7 +3731,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3917,7 +3932,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -4089,7 +4105,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4102,7 +4119,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -4268,7 +4286,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4281,7 +4300,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out index 5151b17955..415bd29681 100644 --- ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out +++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out @@ -103,7 +103,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT key, count(*) FROM test_table1 GROUP BY key @@ -262,7 +263,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT a.key, a.value FROM ( @@ -385,7 +387,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT /*+ MAPJOIN(a) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key @@ -522,7 +525,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT /*+ MAPJOIN(a) */ b.value, count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out index 13219ac56d..adcd2703a1 100644 --- ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out +++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.test_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (ds = '2008-04-08', hr) SELECT key2, value, cast(hr as int) FROM diff --git ql/src/test/results/clientpositive/spark/innerjoin.q.out ql/src/test/results/clientpositive/spark/innerjoin.q.out index 9328b99b0f..c02aa2a1e1 100644 --- ql/src/test/results/clientpositive/spark/innerjoin.q.out +++ ql/src/test/results/clientpositive/spark/innerjoin.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 INNER JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/input12.q.out ql/src/test/results/clientpositive/spark/input12.q.out index 2efd81b86a..13d518876c 100644 --- ql/src/test/results/clientpositive/spark/input12.q.out +++ ql/src/test/results/clientpositive/spark/input12.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -123,7 +124,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -139,7 +141,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 diff --git ql/src/test/results/clientpositive/spark/input13.q.out ql/src/test/results/clientpositive/spark/input13.q.out index 09c7959bfb..1ffd23d9f6 100644 --- ql/src/test/results/clientpositive/spark/input13.q.out +++ ql/src/test/results/clientpositive/spark/input13.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -140,7 +141,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -156,7 +158,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator diff --git ql/src/test/results/clientpositive/spark/input14.q.out ql/src/test/results/clientpositive/spark/input14.q.out index 36f162eb90..ef2f37cf70 100644 --- ql/src/test/results/clientpositive/spark/input14.q.out +++ ql/src/test/results/clientpositive/spark/input14.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/spark/input17.q.out ql/src/test/results/clientpositive/spark/input17.q.out index d95dbcb61a..5f207aa58e 100644 --- ql/src/test/results/clientpositive/spark/input17.q.out +++ ql/src/test/results/clientpositive/spark/input17.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src_thrift diff --git ql/src/test/results/clientpositive/spark/input18.q.out ql/src/test/results/clientpositive/spark/input18.q.out index 65850b2ef5..ba77d156ec 100644 --- ql/src/test/results/clientpositive/spark/input18.q.out +++ ql/src/test/results/clientpositive/spark/input18.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/spark/input1_limit.q.out ql/src/test/results/clientpositive/spark/input1_limit.q.out index dd49287a2c..cfc96cd563 100644 --- ql/src/test/results/clientpositive/spark/input1_limit.q.out +++ ql/src/test/results/clientpositive/spark/input1_limit.q.out @@ -133,7 +133,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -146,7 +147,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 LIMIT 10 diff --git ql/src/test/results/clientpositive/spark/input_part2.q.out ql/src/test/results/clientpositive/spark/input_part2.q.out index e1582e2875..bf53587990 100644 --- ql/src/test/results/clientpositive/spark/input_part2.q.out +++ ql/src/test/results/clientpositive/spark/input_part2.q.out @@ -254,7 +254,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -286,7 +287,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart diff --git ql/src/test/results/clientpositive/spark/insert_into1.q.out ql/src/test/results/clientpositive/spark/insert_into1.q.out index dff389db9c..4a5c3b804d 100644 --- ql/src/test/results/clientpositive/spark/insert_into1.q.out +++ ql/src/test/results/clientpositive/spark/insert_into1.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -188,7 +189,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -302,7 +304,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 10 PREHOOK: type: QUERY @@ -396,7 +399,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table insert_into1 select 1, 'a' PREHOOK: type: QUERY @@ -452,7 +456,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into insert_into1 select 2, 'b' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/insert_into2.q.out ql/src/test/results/clientpositive/spark/insert_into2.q.out index 329387dd91..6724963896 100644 --- ql/src/test/results/clientpositive/spark/insert_into2.q.out +++ ql/src/test/results/clientpositive/spark/insert_into2.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into2 PARTITION (ds='1') SELECT * FROM src order by key limit 100 PREHOOK: type: QUERY @@ -235,7 +236,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 100 @@ -359,7 +361,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 50 diff --git ql/src/test/results/clientpositive/spark/insert_into3.q.out ql/src/test/results/clientpositive/spark/insert_into3.q.out index a6fac2336b..a75cb81652 100644 --- ql/src/test/results/clientpositive/spark/insert_into3.q.out +++ ql/src/test/results/clientpositive/spark/insert_into3.q.out @@ -125,7 +125,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -138,7 +139,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 @@ -274,7 +276,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -287,7 +290,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 diff --git ql/src/test/results/clientpositive/spark/join1.q.out ql/src/test/results/clientpositive/spark/join1.q.out index a0ee4ea5b8..d89d3e5891 100644 --- ql/src/test/results/clientpositive/spark/join1.q.out +++ ql/src/test/results/clientpositive/spark/join1.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/join14.q.out ql/src/test/results/clientpositive/spark/join14.q.out index e804a1d9a5..11edde04be 100644 --- ql/src/test/results/clientpositive/spark/join14.q.out +++ ql/src/test/results/clientpositive/spark/join14.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/spark/join17.q.out ql/src/test/results/clientpositive/spark/join17.q.out index 3644efdde1..e8619feb0b 100644 --- ql/src/test/results/clientpositive/spark/join17.q.out +++ ql/src/test/results/clientpositive/spark/join17.q.out @@ -251,7 +251,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/spark/join2.q.out ql/src/test/results/clientpositive/spark/join2.q.out index f684beb7b9..d2d6b1d992 100644 --- ql/src/test/results/clientpositive/spark/join2.q.out +++ ql/src/test/results/clientpositive/spark/join2.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/join25.q.out ql/src/test/results/clientpositive/spark/join25.q.out index 05e5e701f9..0514466468 100644 --- ql/src/test/results/clientpositive/spark/join25.q.out +++ ql/src/test/results/clientpositive/spark/join25.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/spark/join26.q.out ql/src/test/results/clientpositive/spark/join26.q.out index ccd4526707..e8186abf24 100644 --- ql/src/test/results/clientpositive/spark/join26.q.out +++ ql/src/test/results/clientpositive/spark/join26.q.out @@ -329,7 +329,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join27.q.out ql/src/test/results/clientpositive/spark/join27.q.out index e10d2fb04a..40a91e7d22 100644 --- ql/src/test/results/clientpositive/spark/join27.q.out +++ ql/src/test/results/clientpositive/spark/join27.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/spark/join28.q.out ql/src/test/results/clientpositive/spark/join28.q.out index 8d4d870f89..4a3343adbf 100644 --- ql/src/test/results/clientpositive/spark/join28.q.out +++ ql/src/test/results/clientpositive/spark/join28.q.out @@ -136,7 +136,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq.key1, z.value diff --git ql/src/test/results/clientpositive/spark/join29.q.out ql/src/test/results/clientpositive/spark/join29.q.out index 88929fe2c9..8477a7ec98 100644 --- ql/src/test/results/clientpositive/spark/join29.q.out +++ ql/src/test/results/clientpositive/spark/join29.q.out @@ -136,7 +136,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq1.key, subq1.cnt, subq2.cnt diff --git ql/src/test/results/clientpositive/spark/join3.q.out ql/src/test/results/clientpositive/spark/join3.q.out index e50f091277..5ee3c52be8 100644 --- ql/src/test/results/clientpositive/spark/join3.q.out +++ ql/src/test/results/clientpositive/spark/join3.q.out @@ -114,7 +114,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/join30.q.out ql/src/test/results/clientpositive/spark/join30.q.out index 23650ffe1b..5bfa57b393 100644 --- ql/src/test/results/clientpositive/spark/join30.q.out +++ ql/src/test/results/clientpositive/spark/join30.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/spark/join31.q.out ql/src/test/results/clientpositive/spark/join31.q.out index 4edb4ef976..7eaaff904a 100644 --- ql/src/test/results/clientpositive/spark/join31.q.out +++ ql/src/test/results/clientpositive/spark/join31.q.out @@ -143,7 +143,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq1.key, count(1) as cnt diff --git ql/src/test/results/clientpositive/spark/join32.q.out ql/src/test/results/clientpositive/spark/join32.q.out index 3521f2b0e9..80e8f35f67 100644 --- ql/src/test/results/clientpositive/spark/join32.q.out +++ ql/src/test/results/clientpositive/spark/join32.q.out @@ -335,7 +335,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join32_lessSize.q.out ql/src/test/results/clientpositive/spark/join32_lessSize.q.out index 803b4b1616..63055daede 100644 --- ql/src/test/results/clientpositive/spark/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/spark/join32_lessSize.q.out @@ -343,7 +343,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -880,7 +881,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -1330,7 +1332,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -1778,7 +1781,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -2035,7 +2039,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, x.value, res.value @@ -2279,7 +2284,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, y.value, res.value diff --git ql/src/test/results/clientpositive/spark/join33.q.out ql/src/test/results/clientpositive/spark/join33.q.out index 3521f2b0e9..80e8f35f67 100644 --- ql/src/test/results/clientpositive/spark/join33.q.out +++ ql/src/test/results/clientpositive/spark/join33.q.out @@ -335,7 +335,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join34.q.out ql/src/test/results/clientpositive/spark/join34.q.out index 503235f607..2c9fb99b9e 100644 --- ql/src/test/results/clientpositive/spark/join34.q.out +++ ql/src/test/results/clientpositive/spark/join34.q.out @@ -337,7 +337,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join35.q.out ql/src/test/results/clientpositive/spark/join35.q.out index 54f68af58a..86ccc5adc1 100644 --- ql/src/test/results/clientpositive/spark/join35.q.out +++ ql/src/test/results/clientpositive/spark/join35.q.out @@ -379,7 +379,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join36.q.out ql/src/test/results/clientpositive/spark/join36.q.out index b1717e02e3..b469fc306e 100644 --- ql/src/test/results/clientpositive/spark/join36.q.out +++ ql/src/test/results/clientpositive/spark/join36.q.out @@ -138,7 +138,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt diff --git ql/src/test/results/clientpositive/spark/join37.q.out ql/src/test/results/clientpositive/spark/join37.q.out index 327e93ecd2..8dc4078732 100644 --- ql/src/test/results/clientpositive/spark/join37.q.out +++ ql/src/test/results/clientpositive/spark/join37.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/spark/join39.q.out ql/src/test/results/clientpositive/spark/join39.q.out index 8f0ba62f2b..c094dd9d35 100644 --- ql/src/test/results/clientpositive/spark/join39.q.out +++ ql/src/test/results/clientpositive/spark/join39.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value diff --git ql/src/test/results/clientpositive/spark/join4.q.out ql/src/test/results/clientpositive/spark/join4.q.out index ae91a49789..b42f86c806 100644 --- ql/src/test/results/clientpositive/spark/join4.q.out +++ ql/src/test/results/clientpositive/spark/join4.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join5.q.out ql/src/test/results/clientpositive/spark/join5.q.out index d79af77f9e..d87eabbb9b 100644 --- ql/src/test/results/clientpositive/spark/join5.q.out +++ ql/src/test/results/clientpositive/spark/join5.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join6.q.out ql/src/test/results/clientpositive/spark/join6.q.out index 8ae5e3af59..55ea3d20d8 100644 --- ql/src/test/results/clientpositive/spark/join6.q.out +++ ql/src/test/results/clientpositive/spark/join6.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join7.q.out ql/src/test/results/clientpositive/spark/join7.q.out index 45c42319c3..c784ac98f9 100644 --- ql/src/test/results/clientpositive/spark/join7.q.out +++ ql/src/test/results/clientpositive/spark/join7.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join8.q.out ql/src/test/results/clientpositive/spark/join8.q.out index 4ddad76bda..20efb34e07 100644 --- ql/src/test/results/clientpositive/spark/join8.q.out +++ ql/src/test/results/clientpositive/spark/join8.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join9.q.out ql/src/test/results/clientpositive/spark/join9.q.out index c7a191a4c0..76534ad20b 100644 --- ql/src/test/results/clientpositive/spark/join9.q.out +++ ql/src/test/results/clientpositive/spark/join9.q.out @@ -252,7 +252,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/spark/join_map_ppr.q.out ql/src/test/results/clientpositive/spark/join_map_ppr.q.out index 86f3d9a8c4..d2c674e7c1 100644 --- ql/src/test/results/clientpositive/spark/join_map_ppr.q.out +++ ql/src/test/results/clientpositive/spark/join_map_ppr.q.out @@ -319,7 +319,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -811,7 +812,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out index cdd5c661d7..35dabc1021 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out @@ -153,7 +153,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out index 08df5d3772..db365be68f 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out @@ -204,7 +204,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out index 5bf7f2884e..8a6a60d8a5 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out @@ -114,7 +114,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -130,7 +131,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out index 5ec6d68ea3..f847030f8b 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.nzhang_part10 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part10 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' diff --git ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out index 7f403d575d..e3ed64a444 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.nzhang_part13 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part13 partition (ds="2010-03-03", hr) select * from ( diff --git ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out index ce65e718d9..2a80130a9b 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out @@ -136,7 +136,8 @@ STAGE PLANS: name: default.nzhang_part14 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( diff --git ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out index d066b3ae11..a9cd30e43b 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.nzhang_part_bucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part_bucket partition (ds='2010-03-23', hr) select key, value, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out index d120963697..fa07b227bf 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.nzhang_part3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part3 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out index 7ec76b5c77..c6c851d1bb 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.nzhang_part4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part4 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out index b132a591d4..e55fd53d7c 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.nzhang_part5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part5 partition (value) select key, value from src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out index e19a986952..1846ef4045 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out @@ -367,7 +367,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -399,7 +400,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from srcpart diff --git ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out index 55bcfae4fc..6eed1a7044 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.nzhang_part9 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part9 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/spark/mapreduce1.q.out ql/src/test/results/clientpositive/spark/mapreduce1.q.out index d75b482871..6883343f8f 100644 --- ql/src/test/results/clientpositive/spark/mapreduce1.q.out +++ ql/src/test/results/clientpositive/spark/mapreduce1.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/spark/mapreduce2.q.out ql/src/test/results/clientpositive/spark/mapreduce2.q.out index adfb503aa7..462a1861b5 100644 --- ql/src/test/results/clientpositive/spark/mapreduce2.q.out +++ ql/src/test/results/clientpositive/spark/mapreduce2.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/spark/merge1.q.out ql/src/test/results/clientpositive/spark/merge1.q.out index 8e671e952c..eee05db666 100644 --- ql/src/test/results/clientpositive/spark/merge1.q.out +++ ql/src/test/results/clientpositive/spark/merge1.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -563,7 +564,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -673,7 +675,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/merge2.q.out ql/src/test/results/clientpositive/spark/merge2.q.out index 24116cb468..c210fe4970 100644 --- ql/src/test/results/clientpositive/spark/merge2.q.out +++ ql/src/test/results/clientpositive/spark/merge2.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -563,7 +564,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -673,7 +675,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out index 2891c7df23..75836509bf 100644 --- ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out +++ ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out @@ -419,40 +419,54 @@ STAGE PLANS: PREHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl +PREHOOK: Output: default@stats_tbl #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl +POSTHOOK: Output: default@stats_tbl #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2011 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2011 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2012 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2012 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### PREHOOK: query: explain select count(*), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si) from stats_tbl diff --git ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out index 6376aa79f8..79d9d27203 100644 --- ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out +++ ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out @@ -126,21 +126,29 @@ PREHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2014 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2014 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### PREHOOK: query: explain select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010 diff --git ql/src/test/results/clientpositive/spark/multi_insert.q.out ql/src/test/results/clientpositive/spark/multi_insert.q.out index 33af962c97..741527a288 100644 --- ql/src/test/results/clientpositive/spark/multi_insert.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert.q.out @@ -83,7 +83,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -96,7 +97,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -220,7 +222,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -233,7 +236,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -357,7 +361,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -370,7 +375,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -494,7 +500,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -507,7 +514,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -647,7 +655,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -660,7 +669,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -793,7 +803,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -806,7 +817,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -939,7 +951,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -952,7 +965,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1085,7 +1099,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1098,7 +1113,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1242,7 +1258,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1255,7 +1272,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1425,7 +1443,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1438,7 +1457,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1608,7 +1628,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1621,7 +1642,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1791,7 +1813,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1804,7 +1827,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 diff --git ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out index d8c4b7fc6f..53e28487aa 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out @@ -111,7 +111,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -124,7 +125,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 @@ -296,7 +298,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -309,7 +312,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out index 3d67b1d261..abe331e4e7 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -114,7 +115,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as value from src order by key) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out index 813704f564..96d404cb65 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -130,7 +131,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -227,7 +229,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -240,7 +243,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a INSERT OVERWRITE TABLE e1 @@ -1677,7 +1681,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1690,7 +1695,8 @@ STAGE PLANS: name: default.e3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -1834,7 +1840,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1847,7 +1854,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1860,5 +1868,6 @@ STAGE PLANS: name: default.e3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out index 7b804daba3..903bf1b087 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -172,7 +173,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C @@ -421,7 +423,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -434,7 +437,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -659,7 +663,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -672,7 +677,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -685,7 +691,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -969,7 +976,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -982,7 +990,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -995,7 +1004,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C @@ -1340,7 +1350,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1353,7 +1364,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1366,7 +1378,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1379,7 +1392,8 @@ STAGE PLANS: name: default.src_lv4 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key diff --git ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out index 2b28d5313e..56d7d807be 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out @@ -176,7 +176,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -189,7 +190,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -202,7 +204,8 @@ STAGE PLANS: name: default.src_multi3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select key, count(1) group by key order by key diff --git ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out index 7f1d67b566..94999eda84 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -100,7 +101,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -228,7 +230,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -241,7 +244,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -369,7 +373,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -382,7 +387,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -510,7 +516,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -523,7 +530,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -667,7 +675,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -680,7 +689,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -817,7 +827,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -830,7 +841,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -967,7 +979,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -980,7 +993,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1117,7 +1131,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1130,7 +1145,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1278,7 +1294,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1291,7 +1308,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1465,7 +1483,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1478,7 +1497,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1652,7 +1672,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1665,7 +1686,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1839,7 +1861,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1852,7 +1875,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -2901,7 +2925,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2914,7 +2939,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3121,7 +3147,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3134,7 +3161,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3341,7 +3369,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3354,7 +3383,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3561,7 +3591,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3574,7 +3605,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator diff --git ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out index a146a8e83b..c34d9511db 100644 --- ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out +++ ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -170,7 +171,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -291,7 +293,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -304,7 +307,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -425,7 +429,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -438,7 +443,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -531,7 +537,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -544,7 +551,8 @@ STAGE PLANS: name: default.dest4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -711,7 +719,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -724,7 +733,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -737,5 +747,6 @@ STAGE PLANS: name: default.dest1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/orc_merge1.q.out ql/src/test/results/clientpositive/spark/orc_merge1.q.out index 1407616c01..e5dd9d16f7 100644 --- ql/src/test/results/clientpositive/spark/orc_merge1.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge1.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -179,7 +180,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -302,7 +304,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge2.q.out ql/src/test/results/clientpositive/spark/orc_merge2.q.out index b7f1a65e22..e05ba8c733 100644 --- ql/src/test/results/clientpositive/spark/orc_merge2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge2.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge3.q.out ql/src/test/results/clientpositive/spark/orc_merge3.q.out index 81a6013d1e..2887a4bb8c 100644 --- ql/src/test/results/clientpositive/spark/orc_merge3.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge3.q.out @@ -107,7 +107,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge4.q.out ql/src/test/results/clientpositive/spark/orc_merge4.q.out index 8d433b031a..fdc601204e 100644 --- ql/src/test/results/clientpositive/spark/orc_merge4.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge4.q.out @@ -125,7 +125,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge5.q.out ql/src/test/results/clientpositive/spark/orc_merge5.q.out index 1f8c869574..deb858b2eb 100644 --- ql/src/test/results/clientpositive/spark/orc_merge5.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge5.q.out @@ -69,7 +69,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -166,7 +167,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5b concatenate PREHOOK: type: ALTER_TABLE_MERGE diff --git ql/src/test/results/clientpositive/spark/orc_merge6.q.out ql/src/test/results/clientpositive/spark/orc_merge6.q.out index be62faed0e..25a3eac070 100644 --- ql/src/test/results/clientpositive/spark/orc_merge6.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge6.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (year="2000",hour=24) select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -214,7 +215,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -422,7 +424,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(year="2000",hour=24) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/spark/orc_merge7.q.out ql/src/test/results/clientpositive/spark/orc_merge7.q.out index 01e3eac54a..9f36e95bd6 100644 --- ql/src/test/results/clientpositive/spark/orc_merge7.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge7.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 PREHOOK: type: QUERY @@ -243,7 +244,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -528,7 +530,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out index 1407616c01..e5dd9d16f7 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -179,7 +180,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -302,7 +304,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out index 65790c4962..a3255f456f 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out index 52973c87bb..f86562c170 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 order by userid PREHOOK: type: QUERY @@ -280,7 +281,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/spark/parallel.q.out ql/src/test/results/clientpositive/spark/parallel.q.out index e31fcf0b5a..ecb80622da 100644 --- ql/src/test/results/clientpositive/spark/parallel.q.out +++ ql/src/test/results/clientpositive/spark/parallel.q.out @@ -112,7 +112,8 @@ STAGE PLANS: name: default.src_a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -125,7 +126,8 @@ STAGE PLANS: name: default.src_b Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/spark/parallel_join1.q.out ql/src/test/results/clientpositive/spark/parallel_join1.q.out index 7fdd48d2ca..db2962faa7 100644 --- ql/src/test/results/clientpositive/spark/parallel_join1.q.out +++ ql/src/test/results/clientpositive/spark/parallel_join1.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/parallel_orderby.q.out ql/src/test/results/clientpositive/spark/parallel_orderby.q.out index 483e42d642..de92027d65 100644 --- ql/src/test/results/clientpositive/spark/parallel_orderby.q.out +++ ql/src/test/results/clientpositive/spark/parallel_orderby.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.total_ordered Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table total_ordered as select * from src5 order by key, value PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/pcr.q.out ql/src/test/results/clientpositive/spark/pcr.q.out index 3ed71afc4d..5b5b3aa88f 100644 --- ql/src/test/results/clientpositive/spark/pcr.q.out +++ ql/src/test/results/clientpositive/spark/pcr.q.out @@ -3677,7 +3677,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -3709,7 +3710,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from pcr_t1 @@ -3922,7 +3924,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -3954,7 +3957,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from pcr_t1 diff --git ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out index 12b1724990..1311c99a72 100644 --- ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out +++ ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -182,7 +183,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -198,7 +200,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1442,7 +1445,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1455,7 +1459,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1471,7 +1476,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator diff --git ql/src/test/results/clientpositive/spark/ptf.q.out ql/src/test/results/clientpositive/spark/ptf.q.out index 82fc9f8198..ea599834cd 100644 --- ql/src/test/results/clientpositive/spark/ptf.q.out +++ ql/src/test/results/clientpositive/spark/ptf.q.out @@ -3097,7 +3097,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3110,7 +3111,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part partition by p_mfgr diff --git ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out index 804ff02948..bcf09d87b5 100644 --- ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out +++ ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out @@ -167,7 +167,8 @@ STAGE PLANS: name: default.bucket5_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket5_1 @@ -375,6 +376,7 @@ STAGE PLANS: name: default.complex_tbl_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/sample1.q.out ql/src/test/results/clientpositive/spark/sample1.q.out index ee9eb14685..26d6148ccb 100644 --- ql/src/test/results/clientpositive/spark/sample1.q.out +++ ql/src/test/results/clientpositive/spark/sample1.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample2.q.out ql/src/test/results/clientpositive/spark/sample2.q.out index 85266d1bd8..dff572ea09 100644 --- ql/src/test/results/clientpositive/spark/sample2.q.out +++ ql/src/test/results/clientpositive/spark/sample2.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample4.q.out ql/src/test/results/clientpositive/spark/sample4.q.out index 69e7ee94f8..8c4640f01d 100644 --- ql/src/test/results/clientpositive/spark/sample4.q.out +++ ql/src/test/results/clientpositive/spark/sample4.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample5.q.out ql/src/test/results/clientpositive/spark/sample5.q.out index 558b2dbb68..4af424cc4d 100644 --- ql/src/test/results/clientpositive/spark/sample5.q.out +++ ql/src/test/results/clientpositive/spark/sample5.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample6.q.out ql/src/test/results/clientpositive/spark/sample6.q.out index 4b358291e4..1b49bd32df 100644 --- ql/src/test/results/clientpositive/spark/sample6.q.out +++ ql/src/test/results/clientpositive/spark/sample6.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample7.q.out ql/src/test/results/clientpositive/spark/sample7.q.out index eae33cad36..98c2d92b3a 100644 --- ql/src/test/results/clientpositive/spark/sample7.q.out +++ ql/src/test/results/clientpositive/spark/sample7.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/skewjoin.q.out ql/src/test/results/clientpositive/spark/skewjoin.q.out index b0b28c3114..37fc0d156b 100644 --- ql/src/test/results/clientpositive/spark/skewjoin.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin.q.out @@ -209,7 +209,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out index 3f10ee5685..a0adab1e68 100644 --- ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out @@ -163,7 +163,8 @@ STAGE PLANS: name: default.noskew Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table noskew as select a.* from src a join src b on a.key=b.key order by a.key limit 30 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out index 9424361c56..054905defd 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out @@ -213,7 +213,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out index d0bb917eea..6f1ab067cb 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' @@ -444,7 +445,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out index 6ed3c21a99..90d0e0ecfd 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' @@ -267,7 +268,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' and a.key = 238 @@ -385,7 +387,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table2 a WHERE a.ds = '2' diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out index fb2501597d..71267d6ac4 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out index 1a9118d378..0a63bfb42f 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -162,7 +163,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -244,7 +246,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -327,7 +330,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -410,7 +414,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -492,5 +497,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out index d8bdef2d25..e4681babac 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 @@ -218,7 +219,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out index 7cf3cf7e64..745e221188 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out @@ -140,7 +140,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -1312,7 +1313,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -2500,7 +2502,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 @@ -2604,7 +2607,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out index 622b950cef..e2f68a02bc 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out @@ -661,7 +661,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out index 450499496a..0c29607e2e 100644 --- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out @@ -106,7 +106,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table srcpart_date as select ds as ds, ds as `date` from srcpart group by ds PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index 09af066ad0..fee4956442 100644 --- ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -46,7 +46,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -79,7 +79,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (GROUP) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -3194,7 +3194,7 @@ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-3 Create Table Operator: name:default.nzhang_CTAS1 @@ -3244,7 +3244,7 @@ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-3 Create Table Operator: name:default.nzhang_ctas3 @@ -4806,7 +4806,7 @@ Reducer 6 <- Map 1 (PARTITION-LEVEL SORT) Reducer 7 <- Map 1 (PARTITION-LEVEL SORT) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.part_4"} @@ -4859,7 +4859,7 @@ Stage-3 Output:["_col1","_col2","_col5","_col7"] <- Please refer to the previous Map 1 [PARTITION-LEVEL SORT] Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.part_5"} @@ -5222,7 +5222,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0(CONDITIONAL) Move Operator table:{"name:":"default.dest_j1"} diff --git ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out index 371e756eaf..a5068f8d03 100644 --- ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out +++ ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -182,7 +183,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key,value from src order by key limit 10) a INSERT OVERWRITE TABLE e1 @@ -306,7 +308,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -319,7 +322,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key,value from src order by key) a INSERT OVERWRITE TABLE e1 @@ -1442,7 +1446,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1455,7 +1460,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key,value from src order by key) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out index 0a3ba5f830..d9ea36f996 100644 --- ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out +++ ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out @@ -230,7 +230,8 @@ STAGE PLANS: name: default.dest Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out index 7cafb52b24..d104162b36 100644 --- ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table srcpart_date stored as orc as select ds as ds, ds as `date` from srcpart group by ds PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/stats0.q.out ql/src/test/results/clientpositive/spark/stats0.q.out index ece5f08fbd..2b3255ca20 100644 --- ql/src/test/results/clientpositive/spark/stats0.q.out +++ ql/src/test/results/clientpositive/spark/stats0.q.out @@ -147,7 +147,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table stats_non_partitioned @@ -737,7 +738,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src @@ -1465,7 +1467,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table stats_non_partitioned @@ -2055,7 +2058,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src diff --git ql/src/test/results/clientpositive/spark/stats1.q.out ql/src/test/results/clientpositive/spark/stats1.q.out index 13690dbbba..69d7b4210c 100644 --- ql/src/test/results/clientpositive/spark/stats1.q.out +++ ql/src/test/results/clientpositive/spark/stats1.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE tmptable SELECT unionsrc.key, unionsrc.value diff --git ql/src/test/results/clientpositive/spark/stats10.q.out ql/src/test/results/clientpositive/spark/stats10.q.out index 9c682fce76..8c0610f455 100644 --- ql/src/test/results/clientpositive/spark/stats10.q.out +++ ql/src/test/results/clientpositive/spark/stats10.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') select * from src @@ -383,7 +384,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table bucket3_1 partition (ds) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats12.q.out ql/src/test/results/clientpositive/spark/stats12.q.out index 460cf1d537..19936de7d9 100644 --- ql/src/test/results/clientpositive/spark/stats12.q.out +++ ql/src/test/results/clientpositive/spark/stats12.q.out @@ -153,8 +153,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=12 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats13.q.out ql/src/test/results/clientpositive/spark/stats13.q.out index 859bb6ba99..2058e74deb 100644 --- ql/src/test/results/clientpositive/spark/stats13.q.out +++ ql/src/test/results/clientpositive/spark/stats13.q.out @@ -107,8 +107,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=11 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats2.q.out ql/src/test/results/clientpositive/spark/stats2.q.out index 0272b097e5..49b3da843b 100644 --- ql/src/test/results/clientpositive/spark/stats2.q.out +++ ql/src/test/results/clientpositive/spark/stats2.q.out @@ -140,7 +140,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_t1 partition (ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats3.q.out ql/src/test/results/clientpositive/spark/stats3.q.out index 2f76d0e21d..76f3fb22f3 100644 --- ql/src/test/results/clientpositive/spark/stats3.q.out +++ ql/src/test/results/clientpositive/spark/stats3.q.out @@ -54,7 +54,8 @@ STAGE PLANS: name: default.hive_test_src Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src PREHOOK: type: LOAD diff --git ql/src/test/results/clientpositive/spark/stats5.q.out ql/src/test/results/clientpositive/spark/stats5.q.out index 95293c0071..bb28bdb6dd 100644 --- ql/src/test/results/clientpositive/spark/stats5.q.out +++ ql/src/test/results/clientpositive/spark/stats5.q.out @@ -30,7 +30,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_src compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats7.q.out ql/src/test/results/clientpositive/spark/stats7.q.out index 3605f38a1f..75b95a8947 100644 --- ql/src/test/results/clientpositive/spark/stats7.q.out +++ ql/src/test/results/clientpositive/spark/stats7.q.out @@ -53,7 +53,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats8.q.out ql/src/test/results/clientpositive/spark/stats8.q.out index 4bcbcfde68..971290aa58 100644 --- ql/src/test/results/clientpositive/spark/stats8.q.out +++ ql/src/test/results/clientpositive/spark/stats8.q.out @@ -53,7 +53,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY @@ -168,7 +169,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics PREHOOK: type: QUERY @@ -242,7 +244,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics PREHOOK: type: QUERY @@ -316,7 +319,8 @@ STAGE PLANS: Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics PREHOOK: type: QUERY @@ -390,7 +394,8 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats9.q.out ql/src/test/results/clientpositive/spark/stats9.q.out index beb025c635..fa602f1312 100644 --- ql/src/test/results/clientpositive/spark/stats9.q.out +++ ql/src/test/results/clientpositive/spark/stats9.q.out @@ -36,7 +36,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11603 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcbucket compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out index ad2ca94baa..a36a0b9a20 100644 --- ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out +++ ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out @@ -44,7 +44,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan PREHOOK: type: QUERY @@ -315,7 +316,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats_only_null.q.out ql/src/test/results/clientpositive/spark/stats_only_null.q.out index c73d97e1e9..f6c853fd82 100644 --- ql/src/test/results/clientpositive/spark/stats_only_null.q.out +++ ql/src/test/results/clientpositive/spark/stats_only_null.q.out @@ -181,30 +181,40 @@ STAGE PLANS: PREHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null +PREHOOK: Output: default@stats_null #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null +POSTHOOK: Output: default@stats_null #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2010 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2010 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2011 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2011 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition (dt='2010') PREHOOK: type: DESCTABLE @@ -376,12 +386,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=1 PREHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=1 +PREHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=1 POSTHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=1 +POSTHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition(dt = 1) a PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out index 50096a5a8d..403bc28cfa 100644 --- ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out +++ ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out @@ -89,7 +89,8 @@ STAGE PLANS: Partial Scan Statistics Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out index a0adcaee08..a780301962 100644 --- ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out @@ -257,7 +257,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -270,7 +271,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Shuffle Join JOIN[31][tables = [sq_2_notin_nullcheck]] in Work 'Reducer 2' is a cross product PREHOOK: query: from src b @@ -312,8 +314,8 @@ POSTHOOK: Lineage: src_5.value SIMPLE [(src)b.FieldSchema(name:value, type:strin RUN: Stage-2:MAPRED RUN: Stage-1:MOVE RUN: Stage-0:MOVE -RUN: Stage-3:STATS -RUN: Stage-4:STATS +RUN: Stage-3:COLUMNSTATS +RUN: Stage-4:COLUMNSTATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 @@ -680,7 +682,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -693,7 +696,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Map Join MAPJOIN[47][bigTable=b] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b @@ -736,8 +740,8 @@ RUN: Stage-5:MAPRED RUN: Stage-2:MAPRED RUN: Stage-0:MOVE RUN: Stage-1:MOVE -RUN: Stage-3:STATS -RUN: Stage-4:STATS +RUN: Stage-3:COLUMNSTATS +RUN: Stage-4:COLUMNSTATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 diff --git ql/src/test/results/clientpositive/spark/temp_table.q.out ql/src/test/results/clientpositive/spark/temp_table.q.out index c2ec3b4e21..342d08c1a2 100644 --- ql/src/test/results/clientpositive/spark/temp_table.q.out +++ ql/src/test/results/clientpositive/spark/temp_table.q.out @@ -52,7 +52,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TEMPORARY TABLE foo AS SELECT * FROM src WHERE key % 2 = 0 PREHOOK: type: CREATETABLE_AS_SELECT @@ -118,7 +119,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TEMPORARY TABLE bar AS SELECT * FROM src WHERE key % 2 = 1 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/union10.q.out ql/src/test/results/clientpositive/spark/union10.q.out index ea1bebb8b7..c83afda688 100644 --- ql/src/test/results/clientpositive/spark/union10.q.out +++ ql/src/test/results/clientpositive/spark/union10.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union12.q.out ql/src/test/results/clientpositive/spark/union12.q.out index 063995660c..7d4ffb51a9 100644 --- ql/src/test/results/clientpositive/spark/union12.q.out +++ ql/src/test/results/clientpositive/spark/union12.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union17.q.out ql/src/test/results/clientpositive/spark/union17.q.out index a967c0763a..bc72b6cc3f 100644 --- ql/src/test/results/clientpositive/spark/union17.q.out +++ ql/src/test/results/clientpositive/spark/union17.q.out @@ -197,7 +197,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -210,7 +211,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/spark/union18.q.out ql/src/test/results/clientpositive/spark/union18.q.out index 653c54c76e..124dbdc87c 100644 --- ql/src/test/results/clientpositive/spark/union18.q.out +++ ql/src/test/results/clientpositive/spark/union18.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -143,7 +144,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/spark/union19.q.out ql/src/test/results/clientpositive/spark/union19.q.out index fe5902f395..4d2d56a5a3 100644 --- ql/src/test/results/clientpositive/spark/union19.q.out +++ ql/src/test/results/clientpositive/spark/union19.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -172,7 +173,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/spark/union22.q.out ql/src/test/results/clientpositive/spark/union22.q.out index 6acaba8f70..9a8e55308e 100644 --- ql/src/test/results/clientpositive/spark/union22.q.out +++ ql/src/test/results/clientpositive/spark/union22.q.out @@ -407,7 +407,8 @@ STAGE PLANS: name: default.dst_union22 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') diff --git ql/src/test/results/clientpositive/spark/union25.q.out ql/src/test/results/clientpositive/spark/union25.q.out index 559b318717..c5a001a441 100644 --- ql/src/test/results/clientpositive/spark/union25.q.out +++ ql/src/test/results/clientpositive/spark/union25.q.out @@ -189,5 +189,6 @@ STAGE PLANS: name: default.tmp_unionall Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/union28.q.out ql/src/test/results/clientpositive/spark/union28.q.out index 7ee06fef70..83c105d569 100644 --- ql/src/test/results/clientpositive/spark/union28.q.out +++ ql/src/test/results/clientpositive/spark/union28.q.out @@ -135,7 +135,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( diff --git ql/src/test/results/clientpositive/spark/union29.q.out ql/src/test/results/clientpositive/spark/union29.q.out index 05c44d1768..3fc229bf7b 100644 --- ql/src/test/results/clientpositive/spark/union29.q.out +++ ql/src/test/results/clientpositive/spark/union29.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( diff --git ql/src/test/results/clientpositive/spark/union30.q.out ql/src/test/results/clientpositive/spark/union30.q.out index 9d827eb814..87ae31153a 100644 --- ql/src/test/results/clientpositive/spark/union30.q.out +++ ql/src/test/results/clientpositive/spark/union30.q.out @@ -170,7 +170,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( diff --git ql/src/test/results/clientpositive/spark/union31.q.out ql/src/test/results/clientpositive/spark/union31.q.out index 10f8bdb242..0b02386f99 100644 --- ql/src/test/results/clientpositive/spark/union31.q.out +++ ql/src/test/results/clientpositive/spark/union31.q.out @@ -235,7 +235,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -248,7 +249,8 @@ STAGE PLANS: name: default.t4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from t1 @@ -488,7 +490,8 @@ STAGE PLANS: name: default.t5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -501,7 +504,8 @@ STAGE PLANS: name: default.t6 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( @@ -772,7 +776,8 @@ STAGE PLANS: name: default.t7 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -785,7 +790,8 @@ STAGE PLANS: name: default.t8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( diff --git ql/src/test/results/clientpositive/spark/union4.q.out ql/src/test/results/clientpositive/spark/union4.q.out index cb8c6a2cb8..f78e82da1a 100644 --- ql/src/test/results/clientpositive/spark/union4.q.out +++ ql/src/test/results/clientpositive/spark/union4.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union6.q.out ql/src/test/results/clientpositive/spark/union6.q.out index 6f61839c6e..91c6b2b90e 100644 --- ql/src/test/results/clientpositive/spark/union6.q.out +++ ql/src/test/results/clientpositive/spark/union6.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union_lateralview.q.out ql/src/test/results/clientpositive/spark/union_lateralview.q.out index fe9afb8f88..8835a9811f 100644 --- ql/src/test/results/clientpositive/spark/union_lateralview.q.out +++ ql/src/test/results/clientpositive/spark/union_lateralview.q.out @@ -196,7 +196,8 @@ STAGE PLANS: name: default.test_union_lateral_view Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view SELECT b.key, d.arr_ele, d.value diff --git ql/src/test/results/clientpositive/spark/union_top_level.q.out ql/src/test/results/clientpositive/spark/union_top_level.q.out index 6adf6c43e5..86e92d2463 100644 --- ql/src/test/results/clientpositive/spark/union_top_level.q.out +++ ql/src/test/results/clientpositive/spark/union_top_level.q.out @@ -497,7 +497,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table union_top as select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a @@ -709,7 +710,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a @@ -913,7 +915,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a diff --git ql/src/test/results/clientpositive/spark/vector_char_4.q.out ql/src/test/results/clientpositive/spark/vector_char_4.q.out index 943a4b1423..80e5357837 100644 --- ql/src/test/results/clientpositive/spark/vector_char_4.q.out +++ ql/src/test/results/clientpositive/spark/vector_char_4.q.out @@ -190,5 +190,6 @@ STAGE PLANS: name: default.char_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out index a93c8332bd..101c751741 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out @@ -186,10 +186,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out index bb922e7f2a..c11406d47a 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out index dbbfd34d37..f8d1ec2425 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out index ffce9e6671..a55250b9f4 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out @@ -201,10 +201,12 @@ POSTHOOK: Output: default@small_alltypesorc_b PREHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_b +PREHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b +POSTHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_b PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out index 4f25253ef7..680ee42bb6 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out @@ -27,10 +27,12 @@ POSTHOOK: Output: default@sorted_mod_4 PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@sorted_mod_4 +PREHOOK: Output: default@sorted_mod_4 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@sorted_mod_4 +POSTHOOK: Output: default@sorted_mod_4 #### A masked pattern was here #### PREHOOK: query: create table small_table stored as orc as select ctinyint, cbigint from alltypesorc limit 100 @@ -57,10 +59,12 @@ POSTHOOK: Output: default@small_table PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_table +PREHOOK: Output: default@small_table #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table +POSTHOOK: Output: default@small_table #### A masked pattern was here #### PREHOOK: query: explain vectorization detail formatted select count(*) from (select s.*, st.* @@ -267,10 +271,12 @@ POSTHOOK: Output: default@mod_8_mod_4 PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Output: default@mod_8_mod_4 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Output: default@mod_8_mod_4 #### A masked pattern was here #### PREHOOK: query: create table small_table2 stored as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100 @@ -297,10 +303,12 @@ POSTHOOK: Output: default@small_table2 PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_table2 +PREHOOK: Output: default@small_table2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table2 +POSTHOOK: Output: default@small_table2 #### A masked pattern was here #### PREHOOK: query: explain vectorization detail formatted select count(*) from (select s.*, st.* diff --git ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out index 1c8e479512..1ab7aee6f7 100644 --- ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out @@ -190,5 +190,6 @@ STAGE PLANS: name: default.varchar_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index b231d2cc0a..540a2c8aa1 100644 --- ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -3982,7 +3982,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3995,7 +3996,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part_orc partition by p_mfgr diff --git ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out index 8cb04c7986..cb53f861b5 100644 --- ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out +++ ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out @@ -29,10 +29,12 @@ POSTHOOK: Output: default@s/c PREHOOK: query: ANALYZE TABLE `s/c` COMPUTE STATISTICS FOR COLUMNS key,value PREHOOK: type: QUERY PREHOOK: Input: default@s/c +PREHOOK: Output: default@s/c #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `s/c` COMPUTE STATISTICS FOR COLUMNS key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@s/c +POSTHOOK: Output: default@s/c #### A masked pattern was here #### PREHOOK: query: SELECT key, value FROM `s/c` WHERE key > 80 AND key < 100 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats0.q.out ql/src/test/results/clientpositive/stats0.q.out index 0476acbed1..d242cd388e 100644 --- ql/src/test/results/clientpositive/stats0.q.out +++ ql/src/test/results/clientpositive/stats0.q.out @@ -144,7 +144,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table stats_non_partitioned @@ -731,7 +732,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src @@ -1470,7 +1472,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -2251,7 +2254,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/stats1.q.out ql/src/test/results/clientpositive/stats1.q.out index 5c6049b2b9..4b5846419d 100644 --- ql/src/test/results/clientpositive/stats1.q.out +++ ql/src/test/results/clientpositive/stats1.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE tmptable SELECT unionsrc.key, unionsrc.value diff --git ql/src/test/results/clientpositive/stats10.q.out ql/src/test/results/clientpositive/stats10.q.out index d1fe47393b..de584d7f0d 100644 --- ql/src/test/results/clientpositive/stats10.q.out +++ ql/src/test/results/clientpositive/stats10.q.out @@ -62,7 +62,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') select * from src @@ -374,7 +375,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table bucket3_1 partition (ds) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats12.q.out ql/src/test/results/clientpositive/stats12.q.out index cebdf67d5c..9dbcf43ebf 100644 --- ql/src/test/results/clientpositive/stats12.q.out +++ ql/src/test/results/clientpositive/stats12.q.out @@ -150,8 +150,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=12 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats13.q.out ql/src/test/results/clientpositive/stats13.q.out index 227dfb5478..6403d0616c 100644 --- ql/src/test/results/clientpositive/stats13.q.out +++ ql/src/test/results/clientpositive/stats13.q.out @@ -104,8 +104,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=11 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats2.q.out ql/src/test/results/clientpositive/stats2.q.out index 29a659f83c..6442235ed9 100644 --- ql/src/test/results/clientpositive/stats2.q.out +++ ql/src/test/results/clientpositive/stats2.q.out @@ -134,7 +134,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_t1 partition (ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats3.q.out ql/src/test/results/clientpositive/stats3.q.out index 2f76d0e21d..76f3fb22f3 100644 --- ql/src/test/results/clientpositive/stats3.q.out +++ ql/src/test/results/clientpositive/stats3.q.out @@ -54,7 +54,8 @@ STAGE PLANS: name: default.hive_test_src Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src PREHOOK: type: LOAD diff --git ql/src/test/results/clientpositive/stats4.q.out ql/src/test/results/clientpositive/stats4.q.out index 8f503a90b8..1545931561 100644 --- ql/src/test/results/clientpositive/stats4.q.out +++ ql/src/test/results/clientpositive/stats4.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -175,7 +176,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-10 Map Reduce diff --git ql/src/test/results/clientpositive/stats5.q.out ql/src/test/results/clientpositive/stats5.q.out index 74ddadba04..a1581855f8 100644 --- ql/src/test/results/clientpositive/stats5.q.out +++ ql/src/test/results/clientpositive/stats5.q.out @@ -27,7 +27,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_src compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats7.q.out ql/src/test/results/clientpositive/stats7.q.out index 4d30413b7f..e14b6b79fe 100644 --- ql/src/test/results/clientpositive/stats7.q.out +++ ql/src/test/results/clientpositive/stats7.q.out @@ -50,7 +50,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats8.q.out ql/src/test/results/clientpositive/stats8.q.out index ea5f1d0a5c..cfae7601af 100644 --- ql/src/test/results/clientpositive/stats8.q.out +++ ql/src/test/results/clientpositive/stats8.q.out @@ -50,7 +50,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY @@ -162,7 +163,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics PREHOOK: type: QUERY @@ -233,7 +235,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics PREHOOK: type: QUERY @@ -304,7 +307,8 @@ STAGE PLANS: Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics PREHOOK: type: QUERY @@ -375,7 +379,8 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats9.q.out ql/src/test/results/clientpositive/stats9.q.out index a073b8bfcb..226693360b 100644 --- ql/src/test/results/clientpositive/stats9.q.out +++ ql/src/test/results/clientpositive/stats9.q.out @@ -33,7 +33,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11603 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcbucket compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out index b855b3896e..54566546bb 100644 --- ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out +++ ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/stats_invalidation.q.out ql/src/test/results/clientpositive/stats_invalidation.q.out index a0e76631a3..a98d98ecfd 100644 --- ql/src/test/results/clientpositive/stats_invalidation.q.out +++ ql/src/test/results/clientpositive/stats_invalidation.q.out @@ -21,10 +21,12 @@ POSTHOOK: Lineage: stats_invalid.value SIMPLE [(src)src.FieldSchema(name:value, PREHOOK: query: analyze table stats_invalid compute statistics for columns key,value PREHOOK: type: QUERY PREHOOK: Input: default@stats_invalid +PREHOOK: Output: default@stats_invalid #### A masked pattern was here #### POSTHOOK: query: analyze table stats_invalid compute statistics for columns key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_invalid +POSTHOOK: Output: default@stats_invalid #### A masked pattern was here #### PREHOOK: query: desc formatted stats_invalid PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/stats_missing_warning.q.out ql/src/test/results/clientpositive/stats_missing_warning.q.out index 0ed70a0960..b90578597b 100644 --- ql/src/test/results/clientpositive/stats_missing_warning.q.out +++ ql/src/test/results/clientpositive/stats_missing_warning.q.out @@ -117,26 +117,32 @@ POSTHOOK: Input: default@missing_stats_t3 PREHOOK: query: ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@missing_stats_t1 +PREHOOK: Output: default@missing_stats_t1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@missing_stats_t1 +POSTHOOK: Output: default@missing_stats_t1 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@missing_stats_t2 +PREHOOK: Output: default@missing_stats_t2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@missing_stats_t2 +POSTHOOK: Output: default@missing_stats_t2 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@missing_stats_t3 +PREHOOK: Output: default@missing_stats_t3 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@missing_stats_t3 +POSTHOOK: Output: default@missing_stats_t3 #### A masked pattern was here #### PREHOOK: query: SELECT COUNT(*) FROM missing_stats_t1 t1 diff --git ql/src/test/results/clientpositive/stats_noscan_1.q.out ql/src/test/results/clientpositive/stats_noscan_1.q.out index ad2ca94baa..a36a0b9a20 100644 --- ql/src/test/results/clientpositive/stats_noscan_1.q.out +++ ql/src/test/results/clientpositive/stats_noscan_1.q.out @@ -44,7 +44,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan PREHOOK: type: QUERY @@ -315,7 +316,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_only_null.q.out ql/src/test/results/clientpositive/stats_only_null.q.out index e7068d3396..dd81e4806e 100644 --- ql/src/test/results/clientpositive/stats_only_null.q.out +++ ql/src/test/results/clientpositive/stats_only_null.q.out @@ -169,30 +169,40 @@ STAGE PLANS: PREHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null +PREHOOK: Output: default@stats_null #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null +POSTHOOK: Output: default@stats_null #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2010 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2010 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2011 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2011 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition (dt='2010') PREHOOK: type: DESCTABLE @@ -364,12 +374,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=1 PREHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=1 +PREHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=1 POSTHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=1 +POSTHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition(dt = 1) a PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/stats_partial_size.q.out ql/src/test/results/clientpositive/stats_partial_size.q.out index 4a4061db7e..09129f2410 100644 --- ql/src/test/results/clientpositive/stats_partial_size.q.out +++ ql/src/test/results/clientpositive/stats_partial_size.q.out @@ -28,10 +28,12 @@ POSTHOOK: Output: default@sample PREHOOK: query: analyze table sample compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@sample +PREHOOK: Output: default@sample #### A masked pattern was here #### POSTHOOK: query: analyze table sample compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@sample +POSTHOOK: Output: default@sample #### A masked pattern was here #### PREHOOK: query: explain select sample_partitioned.x from sample_partitioned, sample where sample.y = sample_partitioned.y PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_partscan_1_23.q.out ql/src/test/results/clientpositive/stats_partscan_1_23.q.out index cf9867d5b7..34f70235cb 100644 --- ql/src/test/results/clientpositive/stats_partscan_1_23.q.out +++ ql/src/test/results/clientpositive/stats_partscan_1_23.q.out @@ -89,7 +89,8 @@ STAGE PLANS: Partial Scan Statistics Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_ppr_all.q.out ql/src/test/results/clientpositive/stats_ppr_all.q.out index 3b9bb30ffa..f19e3f5c2c 100644 --- ql/src/test/results/clientpositive/stats_ppr_all.q.out +++ ql/src/test/results/clientpositive/stats_ppr_all.q.out @@ -46,6 +46,10 @@ PREHOOK: Input: default@ss PREHOOK: Input: default@ss@country=US/year=2015/month=1/day=1 PREHOOK: Input: default@ss@country=US/year=2015/month=1/day=2 PREHOOK: Input: default@ss@country=US/year=2015/month=2/day=1 +PREHOOK: Output: default@ss +PREHOOK: Output: default@ss@country=US/year=2015/month=1/day=1 +PREHOOK: Output: default@ss@country=US/year=2015/month=1/day=2 +PREHOOK: Output: default@ss@country=US/year=2015/month=2/day=1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE ss PARTITION(country,year,month,day) compute statistics for columns POSTHOOK: type: QUERY @@ -53,6 +57,10 @@ POSTHOOK: Input: default@ss POSTHOOK: Input: default@ss@country=US/year=2015/month=1/day=1 POSTHOOK: Input: default@ss@country=US/year=2015/month=1/day=2 POSTHOOK: Input: default@ss@country=US/year=2015/month=2/day=1 +POSTHOOK: Output: default@ss +POSTHOOK: Output: default@ss@country=US/year=2015/month=1/day=1 +POSTHOOK: Output: default@ss@country=US/year=2015/month=1/day=2 +POSTHOOK: Output: default@ss@country=US/year=2015/month=2/day=1 #### A masked pattern was here #### PREHOOK: query: explain select sum(order_amount) from ss where (country="US" and year=2015 and month=2 and day=1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/subquery_multiinsert.q.out ql/src/test/results/clientpositive/subquery_multiinsert.q.out index 28c82b85d2..63fcb3f126 100644 --- ql/src/test/results/clientpositive/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/subquery_multiinsert.q.out @@ -233,7 +233,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -293,7 +294,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b @@ -339,8 +341,8 @@ RUN: Stage-6:MAPRED RUN: Stage-4:MAPRED RUN: Stage-0:MOVE RUN: Stage-1:MOVE -RUN: Stage-7:STATS -RUN: Stage-5:STATS +RUN: Stage-7:COLUMNSTATS +RUN: Stage-5:COLUMNSTATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 @@ -699,7 +701,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-16 Map Reduce Local Work @@ -763,7 +766,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Map Reduce @@ -852,8 +856,8 @@ RUN: Stage-4:MAPRED RUN: Stage-12:MAPRED RUN: Stage-1:MOVE RUN: Stage-0:MOVE -RUN: Stage-5:STATS -RUN: Stage-7:STATS +RUN: Stage-5:COLUMNSTATS +RUN: Stage-7:COLUMNSTATS PREHOOK: query: select * from src_4 PREHOOK: type: QUERY PREHOOK: Input: default@src_4 diff --git ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out index 1abe95d31e..5006e85b1b 100644 --- ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out @@ -103,7 +103,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -126,7 +127,8 @@ STAGE PLANS: TableScan alias: uservisits_web_text_none Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.uservisits_web_text_none/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite @@ -162,6 +164,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.uservisits_web_text_none numFiles 1 + numRows 0 + rawDataSize 0 serialization.ddl struct uservisits_web_text_none { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -181,6 +185,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.uservisits_web_text_none numFiles 1 + numRows 0 + rawDataSize 0 serialization.ddl struct uservisits_web_text_none { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -221,7 +227,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.uservisits_web_text_none/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -231,10 +239,12 @@ STAGE PLANS: PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE @@ -245,6 +255,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sourceIP string 0 55 12.763636363636364 13 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -254,6 +265,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector avgTimeOnSite int 1 9 0 9 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -263,6 +275,7 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector adRevenue float 13.099044799804688 492.98870849609375 0 55 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: CREATE TEMPORARY TABLE empty_tab( a int, b double, @@ -338,7 +351,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d, e Column Types: int, double, string, boolean, binary @@ -347,10 +361,12 @@ STAGE PLANS: PREHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e PREHOOK: type: QUERY PREHOOK: Input: default@empty_tab +PREHOOK: Output: default@empty_tab #### A masked pattern was here #### POSTHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e POSTHOOK: type: QUERY POSTHOOK: Input: default@empty_tab +POSTHOOK: Output: default@empty_tab #### A masked pattern was here #### PREHOOK: query: desc formatted empty_tab a PREHOOK: type: DESCTABLE @@ -441,6 +457,7 @@ POSTHOOK: query: desc extended default.UserVisits_web_text_none sourceIP POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@uservisits_web_text_none sourceIP string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -468,14 +485,17 @@ POSTHOOK: Input: default@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sourceIP string 0 55 12.763636363636364 13 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword PREHOOK: type: QUERY PREHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +PREHOOK: Output: test@uservisits_web_text_none POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword POSTHOOK: type: QUERY POSTHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +POSTHOOK: Output: test@uservisits_web_text_none PREHOOK: query: desc extended UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -483,6 +503,7 @@ POSTHOOK: query: desc extended UserVisits_web_text_none sKeyword POSTHOOK: type: DESCTABLE POSTHOOK: Input: test@uservisits_web_text_none sKeyword string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"sKeyword\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -492,6 +513,7 @@ POSTHOOK: Input: test@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sKeyword string 0 54 7.872727272727273 19 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"sKeyword\":\"true\"}} PREHOOK: query: desc formatted test.UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -501,3 +523,4 @@ POSTHOOK: Input: test@uservisits_web_text_none # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment bitVector sKeyword string 0 54 7.872727272727273 19 HL from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"sKeyword\":\"true\"}} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index 59437b80b4..a02a57c00b 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -77,7 +77,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.t @@ -121,7 +121,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.t"} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index 54d1ce3b8e..73c951bd36 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -1127,7 +1127,7 @@ Reducer 8 <- Map 18 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 20 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -1322,13 +1322,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_72] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -1434,7 +1434,7 @@ Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -1673,13 +1673,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_112] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -1742,7 +1742,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -1789,7 +1789,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_11] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -1974,7 +1974,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -2033,7 +2033,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -2092,7 +2092,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -2141,7 +2141,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 7cb8335191..12a13b25bc 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -225,7 +225,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze analyze table src_stats compute statistics POSTHOOK: type: QUERY Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Map 1 TableScan [TS_0] (rows=500/500 width=10) @@ -248,21 +248,19 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-3 - Column Stats Work{} - Stage-2 - Stats-Aggr Operator - Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=1248) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=350) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=350) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] +Stage-2 + Stats Work{} + Stage-0 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=1248) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500/500 width=350) + Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=350) + default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] PREHOOK: query: CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) PREHOOK: type: CREATEMACRO @@ -342,7 +340,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.src_autho_test @@ -653,7 +651,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.orc_merge5"} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index ce943160d7..5df8affefd 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -27,7 +27,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze analyze table src_stats compute statistics POSTHOOK: type: QUERY Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Map 1 TableScan [TS_0] (rows=500/500 width=10) @@ -50,21 +50,19 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-3 - Column Stats Work{} - Stage-2 - Stats-Aggr Operator - Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=1248) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=350) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=350) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] +Stage-2 + Stats Work{} + Stage-0 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=1248) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500/500 width=350) + Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=350) + default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] PREHOOK: query: drop table src_multi2 PREHOOK: type: DROPTABLE @@ -101,66 +99,64 @@ Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Stage-4 - Column Stats Work{} - Stage-3 - Stats-Aggr Operator - Stage-0 - Move Operator - table:{"name:":"default.src_multi2"} - Stage-2 - Dependency Collection{} - Stage-1 - Reducer 5 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - File Output Operator [FS_19] - table:{"name:":"default.src_multi2"} - Select Operator [SEL_18] (rows=849/508 width=178) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_26] (rows=849/508 width=178) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col0","_col3"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_12] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col0"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_2] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - Output:["key","value"] - <-Map 6 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=25/25 width=175) - predicate:key is not null - TableScan [TS_3] (rows=25/25 width=175) - Output:["key","value"] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=849/508 width=178) - Output:["key","value"] - Please refer to the previous Select Operator [SEL_18] +Stage-3 + Stats Work{} + Stage-0 + Move Operator + table:{"name:":"default.src_multi2"} + Stage-2 + Dependency Collection{} + Stage-1 + Reducer 5 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] + <-Reducer 4 [CUSTOM_SIMPLE_EDGE] + File Output Operator [FS_19] + table:{"name:":"default.src_multi2"} + Select Operator [SEL_18] (rows=849/508 width=178) + Output:["_col0","_col1"] + Merge Join Operator [MERGEJOIN_26] (rows=849/508 width=178) + Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col0","_col3"] + <-Map 7 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Select Operator [SEL_14] (rows=500/500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_25] (rows=500/500 width=178) + predicate:key is not null + TableScan [TS_12] (rows=500/500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_15] + PartitionCols:_col0 + Select Operator [SEL_11] (rows=525/319 width=178) + Output:["_col0"] + Group By Operator [GBY_10] (rows=525/319 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Union 2 [SIMPLE_EDGE] + <-Map 1 [CONTAINS] + Reduce Output Operator [RS_9] + PartitionCols:_col0, _col1 + Select Operator [SEL_2] (rows=500/500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_23] (rows=500/500 width=178) + predicate:key is not null + TableScan [TS_0] (rows=500/500 width=178) + Output:["key","value"] + <-Map 6 [CONTAINS] + Reduce Output Operator [RS_9] + PartitionCols:_col0, _col1 + Select Operator [SEL_5] (rows=25/25 width=175) + Output:["_col0","_col1"] + Filter Operator [FIL_24] (rows=25/25 width=175) + predicate:key is not null + TableScan [TS_3] (rows=25/25 width=175) + Output:["key","value"] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=849/508 width=178) + Output:["key","value"] + Please refer to the previous Select Operator [SEL_18] PREHOOK: query: select count(*) from (select * from src union select * from src1)subq PREHOOK: type: QUERY @@ -286,7 +282,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.acid_uami"} @@ -296,17 +292,17 @@ Stage-3 Reducer 2 File Output Operator [FS_8] table:{"name:":"default.acid_uami"} - Select Operator [SEL_4] (rows=4/2 width=302) + Select Operator [SEL_4] (rows=8/2 width=328) Output:["_col0","_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_3] PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=4/2 width=302) + Select Operator [SEL_2] (rows=8/2 width=328) Output:["_col0","_col1","_col3"] - Filter Operator [FIL_9] (rows=4/2 width=226) + Filter Operator [FIL_9] (rows=8/2 width=328) predicate:((de = 109.23) or (de = 119.23)) - TableScan [TS_0] (rows=8/4 width=226) - default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"] + TableScan [TS_0] (rows=8/4 width=328) + default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:NONE,Output:["i","de","vc"] PREHOOK: query: select * from acid_uami order by de PREHOOK: type: QUERY @@ -399,7 +395,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.acid_dot"} diff --git ql/src/test/results/clientpositive/tez/explainuser_3.q.out ql/src/test/results/clientpositive/tez/explainuser_3.q.out index d26a9a32cb..d4812a15c2 100644 --- ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -189,7 +189,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze table src compute statistics POSTHOOK: type: QUERY Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Map 1 TableScan [TS_0] (rows=500 width=10) @@ -202,23 +202,21 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-3 - Column Stats Work{} - Stage-2 - Stats-Aggr Operator - Stage-0 - Reducer 2 - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] +Stage-2 + Stats Work{} + Stage-0 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_3] + Group By Operator [GBY_2] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] + Select Operator [SEL_1] (rows=500 width=178) + Output:["key","value"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: explain CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) @@ -268,7 +266,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.src_autho_test @@ -497,7 +495,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.orc_merge5"} diff --git ql/src/test/results/clientpositive/tunable_ndv.q.out ql/src/test/results/clientpositive/tunable_ndv.q.out index a7839ab70e..5406b6b828 100644 --- ql/src/test/results/clientpositive/tunable_ndv.q.out +++ ql/src/test/results/clientpositive/tunable_ndv.q.out @@ -60,12 +60,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d partition(year=2000) locid PREHOOK: type: DESCTABLE @@ -152,41 +158,57 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_2d locid PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/udf1.q.out ql/src/test/results/clientpositive/udf1.q.out index eebd90f891..b59649be8b 100644 --- ql/src/test/results/clientpositive/udf1.q.out +++ ql/src/test/results/clientpositive/udf1.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf3.q.out ql/src/test/results/clientpositive/udf3.q.out index 96038f12af..e52a827862 100644 --- ql/src/test/results/clientpositive/udf3.q.out +++ ql/src/test/results/clientpositive/udf3.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), min(CAST('' AS INT)), max(CAST('' AS INT)) diff --git ql/src/test/results/clientpositive/udf_10_trims.q.out ql/src/test/results/clientpositive/udf_10_trims.q.out index 3a5303adfe..d15dbc219e 100644 --- ql/src/test/results/clientpositive/udf_10_trims.q.out +++ ql/src/test/results/clientpositive/udf_10_trims.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf_length.q.out ql/src/test/results/clientpositive/udf_length.q.out index fc795bbcf4..e21df147b2 100644 --- ql/src/test/results/clientpositive/udf_length.q.out +++ ql/src/test/results/clientpositive/udf_length.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf_reverse.q.out ql/src/test/results/clientpositive/udf_reverse.q.out index 28b0c9f197..316780e89e 100644 --- ql/src/test/results/clientpositive/udf_reverse.q.out +++ ql/src/test/results/clientpositive/udf_reverse.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/union10.q.out ql/src/test/results/clientpositive/union10.q.out index e14e5e0dd2..f5e2b25801 100644 --- ql/src/test/results/clientpositive/union10.q.out +++ ql/src/test/results/clientpositive/union10.q.out @@ -139,7 +139,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union12.q.out ql/src/test/results/clientpositive/union12.q.out index 10540f9c49..a6cd633e3b 100644 --- ql/src/test/results/clientpositive/union12.q.out +++ ql/src/test/results/clientpositive/union12.q.out @@ -139,7 +139,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union17.q.out ql/src/test/results/clientpositive/union17.q.out index bff29f6f4e..d997c29444 100644 --- ql/src/test/results/clientpositive/union17.q.out +++ ql/src/test/results/clientpositive/union17.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -206,7 +207,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/union18.q.out ql/src/test/results/clientpositive/union18.q.out index 702ff103f9..5f737f68fa 100644 --- ql/src/test/results/clientpositive/union18.q.out +++ ql/src/test/results/clientpositive/union18.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -209,7 +210,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-11 Map Reduce diff --git ql/src/test/results/clientpositive/union19.q.out ql/src/test/results/clientpositive/union19.q.out index 35530912ff..e4ff0d5b74 100644 --- ql/src/test/results/clientpositive/union19.q.out +++ ql/src/test/results/clientpositive/union19.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -178,7 +179,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/union22.q.out ql/src/test/results/clientpositive/union22.q.out index 9134bdf19f..e4b4a0914b 100644 --- ql/src/test/results/clientpositive/union22.q.out +++ ql/src/test/results/clientpositive/union22.q.out @@ -499,7 +499,8 @@ STAGE PLANS: name: default.dst_union22 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 diff --git ql/src/test/results/clientpositive/union25.q.out ql/src/test/results/clientpositive/union25.q.out index a287a97967..9eac674e7d 100644 --- ql/src/test/results/clientpositive/union25.q.out +++ ql/src/test/results/clientpositive/union25.q.out @@ -197,5 +197,6 @@ STAGE PLANS: name: default.tmp_unionall Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/union28.q.out ql/src/test/results/clientpositive/union28.q.out index c3789d08e9..9b203f5461 100644 --- ql/src/test/results/clientpositive/union28.q.out +++ ql/src/test/results/clientpositive/union28.q.out @@ -153,7 +153,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union29.q.out ql/src/test/results/clientpositive/union29.q.out index 87ba275b87..b933426d3f 100644 --- ql/src/test/results/clientpositive/union29.q.out +++ ql/src/test/results/clientpositive/union29.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/union30.q.out ql/src/test/results/clientpositive/union30.q.out index 862ebc11d3..3ee716bffa 100644 --- ql/src/test/results/clientpositive/union30.q.out +++ ql/src/test/results/clientpositive/union30.q.out @@ -188,7 +188,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union31.q.out ql/src/test/results/clientpositive/union31.q.out index b7a63fc6f0..4edaed63b8 100644 --- ql/src/test/results/clientpositive/union31.q.out +++ ql/src/test/results/clientpositive/union31.q.out @@ -193,7 +193,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -236,7 +237,8 @@ STAGE PLANS: name: default.t4 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from t1 @@ -456,7 +458,8 @@ STAGE PLANS: name: default.t5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -469,7 +472,8 @@ STAGE PLANS: name: default.t6 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -782,7 +786,8 @@ STAGE PLANS: name: default.t7 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -795,7 +800,8 @@ STAGE PLANS: name: default.t8 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( diff --git ql/src/test/results/clientpositive/union4.q.out ql/src/test/results/clientpositive/union4.q.out index f3cd59c42e..cdaa78feb5 100644 --- ql/src/test/results/clientpositive/union4.q.out +++ ql/src/test/results/clientpositive/union4.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union6.q.out ql/src/test/results/clientpositive/union6.q.out index fc66cf1254..f7a578c654 100644 --- ql/src/test/results/clientpositive/union6.q.out +++ ql/src/test/results/clientpositive/union6.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union_lateralview.q.out ql/src/test/results/clientpositive/union_lateralview.q.out index f563476de0..c45bf7580f 100644 --- ql/src/test/results/clientpositive/union_lateralview.q.out +++ ql/src/test/results/clientpositive/union_lateralview.q.out @@ -190,7 +190,8 @@ STAGE PLANS: name: default.test_union_lateral_view Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view SELECT b.key, d.arr_ele, d.value diff --git ql/src/test/results/clientpositive/union_stats.q.out ql/src/test/results/clientpositive/union_stats.q.out index 9ea0b519c2..8e68520382 100644 --- ql/src/test/results/clientpositive/union_stats.q.out +++ ql/src/test/results/clientpositive/union_stats.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.t Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/updateAccessTime.q.out ql/src/test/results/clientpositive/updateAccessTime.q.out index 2dcd930b8e..54410ad233 100644 --- ql/src/test/results/clientpositive/updateAccessTime.q.out +++ ql/src/test/results/clientpositive/updateAccessTime.q.out @@ -226,8 +226,10 @@ POSTHOOK: Output: default@src PREHOOK: query: ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value PREHOOK: type: QUERY PREHOOK: Input: default@src +PREHOOK: Output: default@src #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@src +POSTHOOK: Output: default@src #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/vector_bucket.q.out ql/src/test/results/clientpositive/vector_bucket.q.out index 3b74023c2b..057aac873c 100644 --- ql/src/test/results/clientpositive/vector_bucket.q.out +++ ql/src/test/results/clientpositive/vector_bucket.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.non_orc_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: select a, b from non_orc_table order by a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_char_4.q.out ql/src/test/results/clientpositive/vector_char_4.q.out index 1c58fd209b..b3ac2c0871 100644 --- ql/src/test/results/clientpositive/vector_char_4.q.out +++ ql/src/test/results/clientpositive/vector_char_4.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.char_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/vector_groupby4.q.out ql/src/test/results/clientpositive/vector_groupby4.q.out index 34b571e32d..1225a932f7 100644 --- ql/src/test/results/clientpositive/vector_groupby4.q.out +++ ql/src/test/results/clientpositive/vector_groupby4.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1) diff --git ql/src/test/results/clientpositive/vector_groupby6.q.out ql/src/test/results/clientpositive/vector_groupby6.q.out index bc86c15137..539fce1ec4 100644 --- ql/src/test/results/clientpositive/vector_groupby6.q.out +++ ql/src/test/results/clientpositive/vector_groupby6.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1) diff --git ql/src/test/results/clientpositive/vector_multi_insert.q.out ql/src/test/results/clientpositive/vector_multi_insert.q.out index 4013cd4694..8e7c0aa32b 100644 --- ql/src/test/results/clientpositive/vector_multi_insert.q.out +++ ql/src/test/results/clientpositive/vector_multi_insert.q.out @@ -185,7 +185,8 @@ STAGE PLANS: name: default.orc_rn1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -237,7 +238,8 @@ STAGE PLANS: name: default.orc_rn2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-11 Map Reduce @@ -289,7 +291,8 @@ STAGE PLANS: name: default.orc_rn3 Stage: Stage-16 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-17 Map Reduce diff --git ql/src/test/results/clientpositive/vector_outer_join1.q.out ql/src/test/results/clientpositive/vector_outer_join1.q.out index 70bce01c76..32d5581ce7 100644 --- ql/src/test/results/clientpositive/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/vector_outer_join1.q.out @@ -186,10 +186,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_join2.q.out ql/src/test/results/clientpositive/vector_outer_join2.q.out index 2265cb83bc..43f53ae320 100644 --- ql/src/test/results/clientpositive/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/vector_outer_join2.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_join3.q.out ql/src/test/results/clientpositive/vector_outer_join3.q.out index e4e482500f..d5c536c51e 100644 --- ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_join4.q.out ql/src/test/results/clientpositive/vector_outer_join4.q.out index 125ec07353..7858f180b6 100644 --- ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -201,10 +201,12 @@ POSTHOOK: Output: default@small_alltypesorc_b PREHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_b +PREHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b +POSTHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_b PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out index d611103eed..047f9cb707 100644 --- ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out +++ ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out @@ -95,28 +95,34 @@ c1 c2 PREHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_01 +PREHOOK: Output: default@e011_01 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_01 +POSTHOOK: Output: default@e011_01 #### A masked pattern was here #### _c0 _c1 PREHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_02 +PREHOOK: Output: default@e011_02 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_02 +POSTHOOK: Output: default@e011_02 #### A masked pattern was here #### _c0 _c1 PREHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_03 +PREHOOK: Output: default@e011_03 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_03 +POSTHOOK: Output: default@e011_03 #### A masked pattern was here #### _c0 _c1 PREHOOK: query: explain vectorization detail @@ -141,7 +147,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1] @@ -152,7 +158,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) Group By Vectorization: @@ -330,7 +336,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1] @@ -341,7 +347,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 1] - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) Group By Vectorization: @@ -356,7 +362,7 @@ STAGE PLANS: keys: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ @@ -366,7 +372,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -398,7 +404,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -422,7 +428,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -446,7 +452,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -466,14 +472,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -535,19 +541,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -579,7 +585,7 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) Group By Vectorization: @@ -591,7 +597,7 @@ STAGE PLANS: keys: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -615,7 +621,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -647,7 +653,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -671,7 +677,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -695,7 +701,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -715,14 +721,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -788,19 +794,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE TableScan alias: e011_03 Statistics: Num rows: 4 Data size: 36 Basic stats: COMPLETE Column stats: NONE @@ -832,7 +838,7 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) Group By Vectorization: @@ -844,7 +850,7 @@ STAGE PLANS: keys: _col1 (type: decimal(15,2)), _col2 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -868,7 +874,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -900,7 +906,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -924,7 +930,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -948,7 +954,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -968,14 +974,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1040,19 +1046,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -1085,7 +1091,7 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: corr(_col0, _col2) Group By Vectorization: @@ -1097,7 +1103,7 @@ STAGE PLANS: keys: _col1 (type: decimal(15,2)), _col3 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1121,7 +1127,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct) Execution mode: vectorized Map Vectorization: @@ -1153,11 +1159,11 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)), _col2 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1177,14 +1183,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumDouble window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/vector_tablesample_rows.q.out ql/src/test/results/clientpositive/vector_tablesample_rows.q.out index 283762a3eb..873440d4ad 100644 --- ql/src/test/results/clientpositive/vector_tablesample_rows.q.out +++ ql/src/test/results/clientpositive/vector_tablesample_rows.q.out @@ -173,7 +173,8 @@ STAGE PLANS: name: default.decimal_2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -381,7 +382,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/vector_varchar_4.q.out ql/src/test/results/clientpositive/vector_varchar_4.q.out index 205c67a6ae..d0f2065c97 100644 --- ql/src/test/results/clientpositive/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/vector_varchar_4.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.varchar_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/vector_varchar_simple.q.out ql/src/test/results/clientpositive/vector_varchar_simple.q.out index 0f8bdb58c3..5d4c34439a 100644 --- ql/src/test/results/clientpositive/vector_varchar_simple.q.out +++ ql/src/test/results/clientpositive/vector_varchar_simple.q.out @@ -356,7 +356,8 @@ STAGE PLANS: name: default.varchar_3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_windowing.q.out ql/src/test/results/clientpositive/vector_windowing.q.out index 12cd4ccbaa..e4fcd76ff1 100644 --- ql/src/test/results/clientpositive/vector_windowing.q.out +++ ql/src/test/results/clientpositive/vector_windowing.q.out @@ -5118,7 +5118,8 @@ STAGE PLANS: name: default.part_1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -5345,7 +5346,8 @@ STAGE PLANS: name: default.part_2 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-9 Map Reduce @@ -5491,7 +5493,8 @@ STAGE PLANS: name: default.part_3 Stage: Stage-11 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from part INSERT OVERWRITE TABLE part_1 diff --git ql/src/test/results/clientpositive/vector_windowing_expressions.q.out ql/src/test/results/clientpositive/vector_windowing_expressions.q.out index ecf866a1f6..6ee24a9636 100644 --- ql/src/test/results/clientpositive/vector_windowing_expressions.q.out +++ ql/src/test/results/clientpositive/vector_windowing_expressions.q.out @@ -1617,7 +1617,8 @@ STAGE PLANS: name: default.t1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1630,7 +1631,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select sum(i) over (partition by ts order by i), s from over10k) tt insert overwrite table t1 select * insert overwrite table t2 select * PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_windowing_streaming.q.out ql/src/test/results/clientpositive/vector_windowing_streaming.q.out index 0198e2b9a7..687a072883 100644 --- ql/src/test/results/clientpositive/vector_windowing_streaming.q.out +++ ql/src/test/results/clientpositive/vector_windowing_streaming.q.out @@ -763,7 +763,8 @@ STAGE PLANS: name: default.sD Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table sD ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE as select * from (select ctinyint, cdouble, rank() over(partition by ctinyint order by cdouble) r from alltypesorc) a where r < 5