diff --git accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out index 59bca5024d..f14c3d6f21 100644 --- accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out +++ accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.src_x1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce diff --git common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java index b7dc88c939..bcbfe977dd 100644 --- common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java +++ common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -249,7 +249,8 @@ public void print(Printer printer, int indentFlag, String type, Vertex callingVe // find the right op Op choose = null; for (Op op : this.outputOps) { - if (op.outputVertexName.equals(callingVertex.name)) { + // op.outputVertexName may be null + if (callingVertex.name.equals(op.outputVertexName)) { choose = op; } } diff --git contrib/src/test/results/clientpositive/serde_typedbytes.q.out contrib/src/test/results/clientpositive/serde_typedbytes.q.out index 6876ca8775..c844a70627 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git contrib/src/test/results/clientpositive/serde_typedbytes2.q.out contrib/src/test/results/clientpositive/serde_typedbytes2.q.out index 79cf8fe1e5..c0228aa4ff 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes2.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes2.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git contrib/src/test/results/clientpositive/serde_typedbytes3.q.out contrib/src/test/results/clientpositive/serde_typedbytes3.q.out index fec58ef026..0b7541ac13 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes3.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes3.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git contrib/src/test/results/clientpositive/serde_typedbytes4.q.out contrib/src/test/results/clientpositive/serde_typedbytes4.q.out index 1131478a7b..981ff21bb3 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes4.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes4.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git contrib/src/test/results/clientpositive/serde_typedbytes5.q.out contrib/src/test/results/clientpositive/serde_typedbytes5.q.out index 8d3b95ece8..5a7df3c837 100644 --- contrib/src/test/results/clientpositive/serde_typedbytes5.q.out +++ contrib/src/test/results/clientpositive/serde_typedbytes5.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git data/scripts/q_test_init_src.sql data/scripts/q_test_init_src.sql index 56b44e0a5d..2a62d29e34 100644 --- data/scripts/q_test_init_src.sql +++ data/scripts/q_test_init_src.sql @@ -3,3 +3,8 @@ DROP TABLE IF EXISTS src PURGE; CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src; + +ANALYZE TABLE src COMPUTE STATISTICS; + +ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value; + diff --git hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out index e8927e9d64..efc4a117fb 100644 --- hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out +++ hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.src_x1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index e55b1c257e..daf95c30af 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -208,8 +208,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out index f50f4af817..5349210a78 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out @@ -168,8 +168,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### Stage: Stage-3 Map Reduce diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index 660cebba5f..42b9821196 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -226,8 +226,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out index ba0e83d562..cae1a5b518 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out @@ -176,8 +176,9 @@ STAGE PLANS: name: default.table1 Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### Stage: Stage-3 Map Reduce diff --git itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out index 2ababb1eec..45e6d25b03 100644 --- itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out @@ -245,7 +245,8 @@ STAGE PLANS: name: default.blobstore_table Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: EXPLAIN EXTENDED FROM hdfs_table INSERT OVERWRITE TABLE blobstore_table SELECT hdfs_table.key GROUP BY hdfs_table.key ORDER BY hdfs_table.key @@ -469,8 +470,9 @@ STAGE PLANS: name: default.blobstore_table Stage: Stage-3 - Stats-Aggr Operator - Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ### PREHOOK: query: DROP TABLE hdfs_table PREHOOK: type: DROPTABLE diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java index ad2baa2e26..e8ef4b97d6 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java @@ -44,6 +44,7 @@ public void testMTQueries1() throws Exception { util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true); util.getConf().set("hive.stats.dbclass", "fs"); util.getConf().set("hive.mapred.mode", "nonstrict"); + util.getConf().set("hive.stats.column.autogather", "false"); } boolean success = QTestUtil.queryListRunnerMultiThreaded(qfiles, qts); if (!success) { diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 2edf749399..f5ed735c1e 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -203,6 +203,7 @@ private static void startMiniHS2(HiveConf conf) throws Exception { private static void startMiniHS2(HiveConf conf, boolean httpMode) throws Exception { conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); + conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false); MiniHS2.Builder builder = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false); if (httpMode) { builder = builder.withHTTPTransport(); diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index a08163823c..16f784eac3 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -112,6 +112,9 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ auto_sortmerge_join_7.q,\ auto_sortmerge_join_8.q,\ auto_sortmerge_join_9.q,\ + autoColumnStats_1.q,\ + autoColumnStats_10.q,\ + autoColumnStats_2.q,\ bucket2.q,\ bucket3.q,\ bucket4.q,\ @@ -477,8 +480,6 @@ minillaplocal.query.files=\ auto_sortmerge_join_6.q,\ auto_sortmerge_join_8.q,\ auto_sortmerge_join_9.q,\ - autoColumnStats_1.q,\ - autoColumnStats_2.q,\ bucket4.q,\ bucket_groupby.q,\ bucket_many.q,\ diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 65551ad169..fb7d418576 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hive.common.util.HiveStringUtils; import java.io.IOException; @@ -251,7 +250,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, part.setDbName(newDbName); part.setTableName(newTblName); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, - part.getValues(), part.getSd().getCols(), oldt, part); + part.getValues(), part.getSd().getCols(), oldt, part, null); if (colStats != null) { columnStatsNeedUpdated.put(part, colStats); } @@ -290,7 +289,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, List oldCols = part.getSd().getCols(); part.getSd().setCols(newt.getSd().getCols()); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, dbname, name, - part.getValues(), oldCols, oldt, part); + part.getValues(), oldCols, oldt, part, null); assert(colStats == null); msdb.alterPartition(dbname, name, part.getValues(), part); } @@ -433,7 +432,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String // PartitionView does not have SD. We do not need update its column stats if (oldPart.getSd() != null) { updateOrGetPartitionColumnStats(msdb, dbname, name, new_part.getValues(), - oldPart.getSd().getCols(), tbl, new_part); + oldPart.getSd().getCols(), tbl, new_part, null); } msdb.alterPartition(dbname, name, new_part.getValues(), new_part); if (transactionalListeners != null && !transactionalListeners.isEmpty()) { @@ -560,7 +559,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()); ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, dbname, name, oldPart.getValues(), - oldPart.getSd().getCols(), tbl, new_part); + oldPart.getSd().getCols(), tbl, new_part, null); msdb.alterPartition(dbname, name, part_vals, new_part); if (cs != null) { cs.getStatsDesc().setPartName(newPartName); @@ -658,7 +657,7 @@ public Partition alterPartition(final RawStore msdb, Warehouse wh, final String // PartitionView does not have SD and we do not need to update its column stats if (oldTmpPart.getSd() != null) { updateOrGetPartitionColumnStats(msdb, dbname, name, oldTmpPart.getValues(), - oldTmpPart.getSd().getCols(), tbl, tmpPart); + oldTmpPart.getSd().getCols(), tbl, tmpPart, null); } } @@ -810,12 +809,14 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa private ColumnStatistics updateOrGetPartitionColumnStats( RawStore msdb, String dbname, String tblname, List partVals, - List oldCols, Table table, Partition part) + List oldCols, Table table, Partition part, List newCols) throws MetaException, InvalidObjectException { ColumnStatistics newPartsColStats = null; try { - List newCols = part.getSd() == null ? - new ArrayList() : part.getSd().getCols(); + // if newCols are not specified, use default ones. + if (newCols == null) { + newCols = part.getSd() == null ? new ArrayList() : part.getSd().getCols(); + } String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals); String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname) diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 6783e5bc96..1fa65bf84d 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -33,6 +33,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -61,6 +62,7 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; + import org.apache.commons.cli.OptionBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -6932,39 +6934,80 @@ public boolean set_aggr_stats_for(SetPartitionsStatsRequest request) if (request.isSetNeedMerge() && request.isNeedMerge()) { // one single call to get all column stats ColumnStatistics csOld = getMS().getTableColumnStatistics(dbName, tableName, colNames); - if (csOld != null && csOld.getStatsObjSize() != 0) { + Table t = getTable(dbName, tableName); + // we first use t.getParameters() to prune the stats + MetaStoreUtils.pruneColumnStats(firstColStats, t.getParameters()); + // we merge those that can be merged + if (csOld != null && csOld.getStatsObjSize() != 0 + && !firstColStats.getStatsObj().isEmpty()) { MetaStoreUtils.mergeColStats(firstColStats, csOld); } + if (!firstColStats.getStatsObj().isEmpty()) { + return update_table_column_statistics(firstColStats); + } else { + LOG.debug("All the column stats are not accurate to merge."); + return true; + } + } else { + // This is the overwrite case, we do not care about the accuracy. + return update_table_column_statistics(firstColStats); } - return update_table_column_statistics(firstColStats); } } else { // partition level column stats merging - List partitionNames = new ArrayList<>(); + List partitions = new ArrayList<>(); + // note that we may have two or more duplicate partition names. + // see autoColumnStats_2.q under TestMiniLlapLocalCliDriver + Map newStatsMap = new HashMap<>(); for (ColumnStatistics csNew : csNews) { - partitionNames.add(csNew.getStatsDesc().getPartName()); + String partName = csNew.getStatsDesc().getPartName(); + if (newStatsMap.containsKey(partName)) { + MetaStoreUtils.mergeColStats(csNew, newStatsMap.get(partName)); + } + newStatsMap.put(partName, csNew); } - Map map = new HashMap<>(); + + Map oldStatsMap = new HashMap<>(); + Map mapToPart = new HashMap<>(); if (request.isSetNeedMerge() && request.isNeedMerge()) { // a single call to get all column stats for all partitions + List partitionNames = new ArrayList<>(); + partitionNames.addAll(newStatsMap.keySet()); List csOlds = getMS().getPartitionColumnStatistics(dbName, tableName, partitionNames, colNames); - if (csNews.size() != csOlds.size()) { + if (newStatsMap.values().size() != csOlds.size()) { // some of the partitions miss stats. LOG.debug("Some of the partitions miss stats."); } for (ColumnStatistics csOld : csOlds) { - map.put(csOld.getStatsDesc().getPartName(), csOld); + oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld); + } + // another single call to get all the partition objects + partitions = getMS().getPartitionsByNames(dbName, tableName, partitionNames); + for (int index = 0; index < partitionNames.size(); index++) { + mapToPart.put(partitionNames.get(index), partitions.get(index)); } } Table t = getTable(dbName, tableName); - for (int index = 0; index < csNews.size(); index++) { - ColumnStatistics csNew = csNews.get(index); - ColumnStatistics csOld = map.get(csNew.getStatsDesc().getPartName()); - if (csOld != null && csOld.getStatsObjSize() != 0) { - MetaStoreUtils.mergeColStats(csNew, csOld); + for (Entry entry : newStatsMap.entrySet()) { + ColumnStatistics csNew = entry.getValue(); + ColumnStatistics csOld = oldStatsMap.get(entry.getKey()); + if (request.isSetNeedMerge() && request.isNeedMerge()) { + // we first use getParameters() to prune the stats + MetaStoreUtils.pruneColumnStats(csNew, mapToPart.get(entry.getKey()).getParameters()); + // we merge those that can be merged + if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) { + MetaStoreUtils.mergeColStats(csNew, csOld); + } + if (!csNew.getStatsObj().isEmpty()) { + ret = ret && updatePartitonColStats(t, csNew); + } else { + LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName() + + " are not accurate to merge."); + } + } else { + ret = ret && updatePartitonColStats(t, csNew); } - ret = ret && updatePartitonColStats(t, csNew); } } return ret; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index ef097acb0f..010d95971c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1747,6 +1747,19 @@ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) return new URLClassLoader(curPath.toArray(new URL[0]), loader); } + public static void pruneColumnStats(ColumnStatistics csNew, Map parameters) { + List list = new ArrayList<>(); + for (int index = 0; index < csNew.getStatsObj().size(); index++) { + ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index); + // canColumnStatsMerge guarantees that it is accurate before we do merge + if (StatsSetupConst.canColumnStatsMerge(parameters, statsObjNew.getColName())) { + list.add(statsObjNew); + } + // in all the other cases, we can not merge + } + csNew.setStatsObj(list); + } + // this function will merge csOld into csNew. public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) throws InvalidObjectException { @@ -1770,13 +1783,20 @@ public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index); ColumnStatisticsObj statsObjOld = map.get(statsObjNew.getColName()); if (statsObjOld != null) { + // because we already confirm that the stats is accurate + // it is impossible that the column types have been changed while the + // column stats is still accurate. + assert (statsObjNew.getStatsData().getSetField() == statsObjOld.getStatsData() + .getSetField()); // If statsObjOld is found, we can merge. ColumnStatsMerger merger = ColumnStatsMergerFactory.getColumnStatsMerger(statsObjNew, statsObjOld); merger.merge(statsObjNew, statsObjOld); } + // If statsObjOld is not found, we just use statsObjNew as it is accurate. list.add(statsObjNew); } + // in all the other cases, we can not merge csNew.setStatsObj(list); } diff --git pom.xml pom.xml index b521cc4273..2829756af1 100644 --- pom.xml +++ pom.xml @@ -927,6 +927,7 @@ true true + target/eclipse/classes Hive ${basedir}/dev-support/eclipse-styles.xml diff --git ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java index 583d3d3893..1789cc3357 100644 --- ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java @@ -18,11 +18,11 @@ package org.apache.hadoop.hive.ql; +import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.NodeUtils; import org.apache.hadoop.hive.ql.exec.NodeUtils.Function; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskRunner; import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; @@ -64,7 +64,7 @@ private Context ctx; private boolean shutdown; - final Map statsTasks = new HashMap(1); + final Map statsTasks = new HashMap<>(1); public DriverContext() { } @@ -191,7 +191,9 @@ public void prepare(QueryPlan plan) { NodeUtils.iterateTask(rootTasks, StatsTask.class, new Function() { @Override public void apply(StatsTask statsTask) { - statsTasks.put(statsTask.getWork().getAggKey(), statsTask); + if (statsTask.getWork().isAggregating()) { + statsTasks.put(statsTask.getWork().getAggKey(), statsTask); + } } }); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 93b967f238..9dc4c4efad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -52,7 +52,6 @@ import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.io.HivePartitioner; -import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.io.RecordUpdater; import org.apache.hadoop.hive.ql.io.StatsProvidingRecordWriter; import org.apache.hadoop.hive.ql.io.StreamingOutputFormat; @@ -907,7 +906,7 @@ public void process(Object row, int tag) throws HiveException { Object recId = ((StructObjectInspector)rowInspector).getStructFieldData(row, recIdField); int bucketProperty = bucketInspector.get(recIdInspector.getStructFieldData(recId, bucketField)); - int bucketNum = + int bucketNum = BucketCodec.determineVersion(bucketProperty).decodeWriterId(bucketProperty); writerOffset = 0; if (multiFileSpray) { @@ -1378,7 +1377,7 @@ private void publishStats() throws HiveException { } StatsCollectionContext sContext = new StatsCollectionContext(hconf); - sContext.setStatsTmpDir(conf.getStatsTmpDir()); + sContext.setStatsTmpDir(conf.getTmpStatsDir()); if (!statsPublisher.connect(sContext)) { // just return, stats gathering should not block the main query LOG.error("StatsPublishing error: cannot connect to database"); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java deleted file mode 100644 index c333c494a1..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java +++ /dev/null @@ -1,396 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import java.io.Serializable; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.HiveStatsUtils; -import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.QueryPlan; -import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.mapred.FileSplit; -import org.apache.hadoop.mapred.InputFormat; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.util.StringUtils; -import org.apache.hive.common.util.ReflectionUtil; - -import com.google.common.collect.Lists; -import com.google.common.collect.MapMaker; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -/** - * StatsNoJobTask is used in cases where stats collection is the only task for the given query (no - * parent MR or Tez job). It is used in the following cases 1) ANALYZE with noscan for - * file formats that implement StatsProvidingRecordReader interface: ORC format (implements - * StatsProvidingRecordReader) stores column statistics for all columns in the file footer. Its much - * faster to compute the table/partition statistics by reading the footer than scanning all the - * rows. This task can be used for computing basic stats like numFiles, numRows, fileSize, - * rawDataSize from ORC footer. - **/ -public class StatsNoJobTask extends Task implements Serializable { - - private static final long serialVersionUID = 1L; - private static transient final Logger LOG = LoggerFactory.getLogger(StatsNoJobTask.class); - private ConcurrentMap partUpdates; - private Table table; - private String tableFullName; - private JobConf jc = null; - - public StatsNoJobTask() { - super(); - } - - @Override - public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext driverContext, - CompilationOpContext opContext) { - super.initialize(queryState, queryPlan, driverContext, opContext); - jc = new JobConf(conf); - } - - @Override - public int execute(DriverContext driverContext) { - - LOG.info("Executing stats (no job) task"); - - String tableName = ""; - ExecutorService threadPool = null; - Hive db = getHive(); - try { - tableName = work.getTableSpecs().tableName; - table = db.getTable(tableName); - int numThreads = HiveConf.getIntVar(conf, ConfVars.HIVE_STATS_GATHER_NUM_THREADS); - tableFullName = table.getFullyQualifiedName(); - threadPool = Executors.newFixedThreadPool(numThreads, - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("StatsNoJobTask-Thread-%d") - .build()); - partUpdates = new MapMaker().concurrencyLevel(numThreads).makeMap(); - LOG.info("Initialized threadpool for stats computation with {} threads", numThreads); - } catch (HiveException e) { - LOG.error("Cannot get table {}", tableName, e); - console.printError("Cannot get table " + tableName, e.toString()); - } - - return aggregateStats(threadPool, db); - } - - @Override - public StageType getType() { - return StageType.STATS; - } - - @Override - public String getName() { - return "STATS-NO-JOB"; - } - - class StatsCollection implements Runnable { - - private final Partition partn; - - public StatsCollection(Partition part) { - this.partn = part; - } - - @Override - public void run() { - - // get the list of partitions - org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); - Map parameters = tPart.getParameters(); - - try { - Path dir = new Path(tPart.getSd().getLocation()); - long numRows = 0; - long rawDataSize = 0; - long fileSize = 0; - long numFiles = 0; - FileSystem fs = dir.getFileSystem(conf); - FileStatus[] fileList = HiveStatsUtils.getFileStatusRecurse(dir, -1, fs); - - boolean statsAvailable = false; - for(FileStatus file: fileList) { - if (!file.isDir()) { - InputFormat inputFormat = ReflectionUtil.newInstance( - partn.getInputFormatClass(), jc); - InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, - new String[] { partn.getLocation() }); - org.apache.hadoop.mapred.RecordReader recordReader = - inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); - StatsProvidingRecordReader statsRR; - if (recordReader instanceof StatsProvidingRecordReader) { - statsRR = (StatsProvidingRecordReader) recordReader; - rawDataSize += statsRR.getStats().getRawDataSize(); - numRows += statsRR.getStats().getRowCount(); - fileSize += file.getLen(); - numFiles += 1; - statsAvailable = true; - } - recordReader.close(); - } - } - - if (statsAvailable) { - parameters.put(StatsSetupConst.ROW_COUNT, String.valueOf(numRows)); - parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(rawDataSize)); - parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(fileSize)); - parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(numFiles)); - - partUpdates.put(tPart.getSd().getLocation(), new Partition(table, tPart)); - - // printout console and debug logs - String threadName = Thread.currentThread().getName(); - String msg = "Partition " + tableFullName + partn.getSpec() + " stats: [" - + toString(parameters) + ']'; - LOG.debug("{}: {}", threadName, msg); - console.printInfo(msg); - } else { - String threadName = Thread.currentThread().getName(); - String msg = "Partition " + tableFullName + partn.getSpec() + " does not provide stats."; - LOG.debug("{}: {}", threadName, msg); - } - } catch (Exception e) { - console.printInfo("[Warning] could not update stats for " + tableFullName + partn.getSpec() - + ".", - "Failed with exception " + e.getMessage() + "\n" + StringUtils.stringifyException(e)); - - // Before updating the partition params, if any partition params is null - // and if statsReliable is true then updatePartition() function will fail - // the task by returning 1 - if (work.isStatsReliable()) { - partUpdates.put(tPart.getSd().getLocation(), null); - } - } - } - - private String toString(Map parameters) { - StringBuilder builder = new StringBuilder(); - for (String statType : StatsSetupConst.supportedStats) { - String value = parameters.get(statType); - if (value != null) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(statType).append('=').append(value); - } - } - return builder.toString(); - } - - } - - private int aggregateStats(ExecutorService threadPool, Hive db) { - int ret = 0; - - try { - Collection partitions = null; - if (work.getPrunedPartitionList() == null) { - partitions = getPartitionsList(); - } else { - partitions = work.getPrunedPartitionList().getPartitions(); - } - - // non-partitioned table - if (partitions == null) { - org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable(); - Map parameters = tTable.getParameters(); - try { - Path dir = new Path(tTable.getSd().getLocation()); - LOG.debug("Aggregating stats for " + dir); - long numRows = 0; - long rawDataSize = 0; - long fileSize = 0; - long numFiles = 0; - FileSystem fs = dir.getFileSystem(conf); - FileStatus[] fileList = HiveStatsUtils.getFileStatusRecurse(dir, -1, fs); - - boolean statsAvailable = false; - for(FileStatus file: fileList) { - LOG.debug("Computing stats for " + file); - if (!file.isDir()) { - InputFormat inputFormat = ReflectionUtil.newInstance( - table.getInputFormatClass(), jc); - InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table - .getDataLocation().toString() }); - if (file.getLen() == 0) { - numFiles += 1; - statsAvailable = true; - } else { - org.apache.hadoop.mapred.RecordReader recordReader = - inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); - StatsProvidingRecordReader statsRR; - if (recordReader instanceof StatsProvidingRecordReader) { - statsRR = (StatsProvidingRecordReader) recordReader; - numRows += statsRR.getStats().getRowCount(); - rawDataSize += statsRR.getStats().getRawDataSize(); - fileSize += file.getLen(); - numFiles += 1; - statsAvailable = true; - } - recordReader.close(); - } - } - } - - if (statsAvailable) { - parameters.put(StatsSetupConst.ROW_COUNT, String.valueOf(numRows)); - parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(rawDataSize)); - parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(fileSize)); - parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(numFiles)); - EnvironmentContext environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); - - db.alterTable(table, environmentContext); - - String msg = "Table " + tableFullName + " stats: [" + toString(parameters) + ']'; - if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { - Utilities.FILE_OP_LOGGER.trace(msg); - } - console.printInfo(msg); - LOG.debug("Table {} does not provide stats.", tableFullName); - } - } catch (Exception e) { - console.printInfo("[Warning] could not update stats for " + tableFullName + ".", - "Failed with exception " + e.getMessage() + "\n" + StringUtils.stringifyException(e)); - } - } else { - - // Partitioned table - for (Partition partn : partitions) { - threadPool.execute(new StatsCollection(partn)); - } - - LOG.debug("Stats collection waiting for threadpool to shutdown.."); - shutdownAndAwaitTermination(threadPool); - LOG.debug("Stats collection threadpool shutdown successful."); - - ret = updatePartitions(db); - } - - } catch (Exception e) { - // Fail the query if the stats are supposed to be reliable - if (work.isStatsReliable()) { - ret = -1; - } - } - - // The return value of 0 indicates success, - // anything else indicates failure - return ret; - } - - private int updatePartitions(Hive db) throws InvalidOperationException, HiveException { - if (!partUpdates.isEmpty()) { - List updatedParts = Lists.newArrayList(partUpdates.values()); - if (updatedParts.contains(null) && work.isStatsReliable()) { - LOG.debug("Stats requested to be reliable. Empty stats found and hence failing the task."); - return -1; - } else { - LOG.debug("Bulk updating partitions.."); - EnvironmentContext environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); - db.alterPartitions(tableFullName, Lists.newArrayList(partUpdates.values()), - environmentContext); - LOG.debug("Bulk updated {} partitions.", partUpdates.values().size()); - } - } - return 0; - } - - private void shutdownAndAwaitTermination(ExecutorService threadPool) { - - // Disable new tasks from being submitted - threadPool.shutdown(); - try { - - // Wait a while for existing tasks to terminate - while (!threadPool.awaitTermination(10, TimeUnit.SECONDS)) { - LOG.debug("Waiting for all stats tasks to finish..."); - } - // Cancel currently executing tasks - threadPool.shutdownNow(); - - // Wait a while for tasks to respond to being cancelled - if (!threadPool.awaitTermination(100, TimeUnit.SECONDS)) { - LOG.debug("Stats collection thread pool did not terminate"); - } - } catch (InterruptedException ie) { - - // Cancel again if current thread also interrupted - threadPool.shutdownNow(); - - // Preserve interrupt status - Thread.currentThread().interrupt(); - } - } - - private String toString(Map parameters) { - StringBuilder builder = new StringBuilder(); - for (String statType : StatsSetupConst.supportedStats) { - String value = parameters.get(statType); - if (value != null) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(statType).append('=').append(value); - } - } - return builder.toString(); - } - - private List getPartitionsList() throws HiveException { - if (work.getTableSpecs() != null) { - TableSpec tblSpec = work.getTableSpecs(); - table = tblSpec.tableHandle; - if (!table.isPartitioned()) { - return null; - } else { - return tblSpec.partitions; - } - } - return null; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index 682b42c202..1a5b4474e3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@ -16,501 +16,145 @@ * limitations under the License. */ - package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; -import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; -import org.apache.hadoop.hive.ql.plan.LoadTableDesc; -import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.api.StageType; -import org.apache.hadoop.hive.ql.stats.StatsAggregator; -import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; -import org.apache.hadoop.hive.ql.stats.StatsFactory; -import org.apache.hadoop.hive.ql.stats.StatsPublisher; -import org.apache.hadoop.hive.ql.stats.StatsUtils; -import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask; +import org.apache.hadoop.hive.ql.stats.BasicStatsTask; +import org.apache.hadoop.hive.ql.stats.ColStatsProcessor; +import org.apache.hadoop.hive.ql.stats.IStatsProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** - * StatsTask implementation. StatsTask mainly deals with "collectable" stats. These are - * stats that require data scanning and are collected during query execution (unless the user - * explicitly requests data scanning just for the purpose of stats computation using the "ANALYZE" - * command. All other stats are computed directly by the MetaStore. The rationale being that the - * MetaStore layer covers all Thrift calls and provides better guarantees about the accuracy of - * those stats. + * StatsTask implementation. **/ -public class StatsTask extends Task implements Serializable { +public class StatsTask extends Task implements Serializable { private static final long serialVersionUID = 1L; private static transient final Logger LOG = LoggerFactory.getLogger(StatsTask.class); - private Table table; - private Collection dpPartSpecs; - public StatsTask() { super(); - dpPartSpecs = null; } - @Override - protected void receiveFeed(FeedType feedType, Object feedValue) { - // this method should be called by MoveTask when there are dynamic partitions generated - if (feedType == FeedType.DYNAMIC_PARTITIONS) { - dpPartSpecs = (Collection) feedValue; - } - } + List processors = new ArrayList<>(); @Override - public int execute(DriverContext driverContext) { - if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { - return 0; - } - LOG.info("Executing stats task"); - // Make sure that it is either an ANALYZE, INSERT OVERWRITE (maybe load) or CTAS command - short workComponentsPresent = 0; - if (work.getLoadTableDesc() != null) { - workComponentsPresent++; + public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx, + CompilationOpContext opContext) { + super.initialize(queryState, queryPlan, ctx, opContext); + + if (work.getBasicStatsWork() != null) { + BasicStatsTask task = new BasicStatsTask(conf, work.getBasicStatsWork()); + task.followedColStats = work.hasColStats(); + processors.add(0, task); } - if (work.getTableSpecs() != null) { - workComponentsPresent++; + if (work.isFooterScan()) { + BasicStatsNoJobTask t = new BasicStatsNoJobTask(conf, work.getBasicStatsNoJobWork()); + processors.add(0, t); } - if (work.getLoadFileDesc() != null) { - workComponentsPresent++; + if (work.hasColStats()) { + processors.add(new ColStatsProcessor(work.getColStats(), conf)); } - assert (workComponentsPresent == 1); - - String tableName = ""; - Hive hive = getHive(); - try { - if (work.getLoadTableDesc() != null) { - tableName = work.getLoadTableDesc().getTable().getTableName(); - } else if (work.getTableSpecs() != null){ - tableName = work.getTableSpecs().tableName; - } else { - tableName = work.getLoadFileDesc().getDestinationCreateTable(); - } - - table = hive.getTable(tableName); - - } catch (HiveException e) { - LOG.error("Cannot get table {}", tableName, e); - console.printError("Cannot get table " + tableName, e.toString()); + for (IStatsProcessor p : processors) { + p.initialize(opContext); } - - return aggregateStats(hive); - } - @Override - public StageType getType() { - return StageType.STATS; - } @Override - public String getName() { - return "STATS"; - } - - private int aggregateStats(Hive db) { - - StatsAggregator statsAggregator = null; + public int execute(DriverContext driverContext) { + if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { + return 0; + } + if (work.isAggregating() && work.isFooterScan()) { + LOG.error("Can not have both basic stats work and stats no job work!"); + return 1; + } int ret = 0; - StatsCollectionContext scc = null; - EnvironmentContext environmentContext = null; try { - // Stats setup: - final Warehouse wh = new Warehouse(conf); - if (!getWork().getNoStatsAggregator() && !getWork().isNoScanAnalyzeCommand()) { - try { - scc = getContext(); - statsAggregator = createStatsAggregator(scc, conf); - } catch (HiveException e) { - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { - throw e; - } - console.printError(ErrorMsg.STATS_SKIPPING_BY_ERROR.getErrorCodedMsg(e.toString())); - } - } - - List partitions = getPartitionsList(db); - boolean atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); - - String tableFullName = table.getFullyQualifiedName(); - - if (partitions == null) { - org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable(); - Map parameters = tTable.getParameters(); - // In the following scenarios, we need to reset the stats to true. - // work.getTableSpecs() != null means analyze command - // work.getLoadTableDesc().getReplace() is true means insert overwrite command - // work.getLoadFileDesc().getDestinationCreateTable().isEmpty() means CTAS etc. - // acidTable will not have accurate stats unless it is set through analyze command. - if (work.getTableSpecs() == null && AcidUtils.isFullAcidTable(table)) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } else if (work.getTableSpecs() != null - || (work.getLoadTableDesc() != null - && (work.getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL)) - || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() - .getDestinationCreateTable().isEmpty())) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); - } - // non-partitioned tables: - if (!existStats(parameters) && atomic) { - return 0; - } - - // The collectable stats for the aggregator needs to be cleared. - // For eg. if a file is being loaded, the old number of rows are not valid - if (work.isClearAggregatorStats()) { - // we choose to keep the invalid stats and only change the setting. - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } - - updateQuickStats(wh, parameters, tTable.getSd()); - if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { - if (statsAggregator != null) { - String prefix = getAggregationPrefix(table, null); - updateStats(statsAggregator, parameters, prefix, atomic); - } - // write table stats to metastore - if (!getWork().getNoStatsAggregator()) { - environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, - StatsSetupConst.TASK); - } - } - - getHive().alterTable(table, environmentContext); - if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { - console.printInfo("Table " + tableFullName + " stats: [" + toString(parameters) + ']'); - } - LOG.info("Table {} stats: [{}]", tableFullName, toString(parameters)); - if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { - Utilities.FILE_OP_LOGGER.trace( - "Table " + tableFullName + " stats: [" + toString(parameters) + ']'); - } - } else { - // Partitioned table: - // Need to get the old stats of the partition - // and update the table stats based on the old and new stats. - List updates = new ArrayList(); - //Get the file status up-front for all partitions. Beneficial in cases of blob storage systems - final Map fileStatusMap = new ConcurrentHashMap(); - int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 1); - // In case thread count is set to 0, use single thread. - poolSize = Math.max(poolSize, 1); - final ExecutorService pool = Executors.newFixedThreadPool(poolSize, - new ThreadFactoryBuilder().setDaemon(true) - .setNameFormat("stats-updater-thread-%d") - .build()); - final List> futures = Lists.newLinkedList(); - LOG.debug("Getting file stats of all partitions. threadpool size: {}", poolSize); - try { - for(final Partition partn : partitions) { - final String partitionName = partn.getName(); - final org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); - Map parameters = tPart.getParameters(); - - if (!existStats(parameters) && atomic) { - continue; - } - futures.add(pool.submit(new Callable() { - @Override - public Void call() throws Exception { - FileStatus[] partfileStatus = wh.getFileStatusesForSD(tPart.getSd()); - fileStatusMap.put(partitionName, partfileStatus); - return null; - } - })); - } - pool.shutdown(); - for(Future future : futures) { - future.get(); - } - } catch (InterruptedException e) { - LOG.debug("Cancelling {} file stats lookup tasks", futures.size()); - //cancel other futures - for (Future future : futures) { - future.cancel(true); - } - // Fail the query if the stats are supposed to be reliable - if (work.isStatsReliable()) { - ret = 1; - } - } finally { - if (pool != null) { - pool.shutdownNow(); - } - LOG.debug("Finished getting file stats of all partitions"); - } - - for (Partition partn : partitions) { - // - // get the old partition stats - // - org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); - Map parameters = tPart.getParameters(); - if (work.getTableSpecs() == null && AcidUtils.isFullAcidTable(table)) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } else if (work.getTableSpecs() != null - || (work.getLoadTableDesc() != null - && (work.getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL)) - || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() - .getDestinationCreateTable().isEmpty())) { - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); - } - //only when the stats exist, it is added to fileStatusMap - if (!fileStatusMap.containsKey(partn.getName())) { - continue; - } - - // The collectable stats for the aggregator needs to be cleared. - // For eg. if a file is being loaded, the old number of rows are not valid - if (work.isClearAggregatorStats()) { - // we choose to keep the invalid stats and only change the setting. - StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); - } + if (work.isFooterScan()) { + work.getBasicStatsNoJobWork().setPartitions(work.getPartitions()); + } - updateQuickStats(parameters, fileStatusMap.get(partn.getName())); - if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { - if (statsAggregator != null) { - String prefix = getAggregationPrefix(table, partn); - updateStats(statsAggregator, parameters, prefix, atomic); - } - if (!getWork().getNoStatsAggregator()) { - environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, - StatsSetupConst.TASK); - } - } - updates.add(new Partition(table, tPart)); + Hive db = getHive(); + Table tbl = getTable(db); - if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { - console.printInfo("Partition " + tableFullName + partn.getSpec() + - " stats: [" + toString(parameters) + ']'); - } - LOG.info("Partition {}{} stats: [{}]", tableFullName, partn.getSpec(), - toString(parameters)); - } - if (!updates.isEmpty()) { - db.alterPartitions(tableFullName, updates, environmentContext); + for (IStatsProcessor task : processors) { + task.setDpPartSpecs(dpPartSpecs); + ret = task.process(db, tbl); + if (ret != 0) { + return ret; } } - } catch (Exception e) { - console.printInfo("[Warning] could not update stats.", - "Failed with exception " + e.getMessage() + "\n" - + StringUtils.stringifyException(e)); - - // Fail the query if the stats are supposed to be reliable - if (work.isStatsReliable()) { - ret = 1; - } - } finally { - if (statsAggregator != null) { - statsAggregator.closeConnection(scc); - } + LOG.error("Failed to run stats task", e); + return 1; } - // The return value of 0 indicates success, - // anything else indicates failure - return ret; + return 0; } - private String getAggregationPrefix(Table table, Partition partition) - throws MetaException { - // prefix is of the form dbName.tblName - String prefix = StatsUtils.getFullyQualifiedTableName(table.getDbName(), - org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(table.getTableName())); - if (partition != null) { - return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); + private Table getTable(Hive db) throws SemanticException, HiveException { + Table tbl = work.getTable(); + // FIXME for ctas this is still needed because location is not set sometimes + if (tbl.getSd().getLocation() == null) { + tbl = db.getTable(work.getFullTableName()); } - return prefix; + return tbl; } - private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { - String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); - StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); - if (factory == null) { - throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); - } - // initialize stats publishing table for noscan which has only stats task - // the rest of MR task following stats task initializes it in ExecDriver.java - StatsPublisher statsPublisher = factory.getStatsPublisher(); - if (!statsPublisher.init(scc)) { // creating stats table if not exists - throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); - } - - // manufacture a StatsAggregator - StatsAggregator statsAggregator = factory.getStatsAggregator(); - if (!statsAggregator.connect(scc)) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); - } - return statsAggregator; - } - - private StatsCollectionContext getContext() throws HiveException { - - StatsCollectionContext scc = new StatsCollectionContext(conf); - Task sourceTask = getWork().getSourceTask(); - if (sourceTask == null) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); - } - scc.setTask(sourceTask); - scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); - return scc; - } - - private boolean existStats(Map parameters) { - return parameters.containsKey(StatsSetupConst.ROW_COUNT) - || parameters.containsKey(StatsSetupConst.NUM_FILES) - || parameters.containsKey(StatsSetupConst.TOTAL_SIZE) - || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE) - || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS); - } - - private void updateStats(StatsAggregator statsAggregator, - Map parameters, String prefix, boolean atomic) - throws HiveException { - - String aggKey = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR; - - for (String statType : StatsSetupConst.statsRequireCompute) { - String value = statsAggregator.aggregateStats(aggKey, statType); - if (value != null && !value.isEmpty()) { - long longValue = Long.parseLong(value); - - if (work.getLoadTableDesc() != null && - (work.getLoadTableDesc().getLoadFileType() != LoadFileType.REPLACE_ALL)) { - String originalValue = parameters.get(statType); - if (originalValue != null) { - longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid - } - } - parameters.put(statType, String.valueOf(longValue)); - } else { - if (atomic) { - throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); - } - } - } + @Override + public StageType getType() { + return StageType.STATS; } - private void updateQuickStats(Warehouse wh, Map parameters, - StorageDescriptor desc) throws MetaException { - /** - * calculate fast statistics - */ - FileStatus[] partfileStatus = wh.getFileStatusesForSD(desc); - updateQuickStats(parameters, partfileStatus); + @Override + public String getName() { + return "STATS TASK"; } - private void updateQuickStats(Map parameters, - FileStatus[] partfileStatus) throws MetaException { - MetaStoreUtils.populateQuickStats(partfileStatus, parameters); - } + private Collection dpPartSpecs; - private String toString(Map parameters) { - StringBuilder builder = new StringBuilder(); - for (String statType : StatsSetupConst.supportedStats) { - String value = parameters.get(statType); - if (value != null) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(statType).append('=').append(value); - } + @Override + protected void receiveFeed(FeedType feedType, Object feedValue) { + // this method should be called by MoveTask when there are dynamic + // partitions generated + if (feedType == FeedType.DYNAMIC_PARTITIONS) { + dpPartSpecs = (Collection) feedValue; } - return builder.toString(); } - /** - * Get the list of partitions that need to update statistics. - * TODO: we should reuse the Partitions generated at compile time - * since getting the list of partitions is quite expensive. - * - * @return a list of partitions that need to update statistics. - * @throws HiveException - */ - private List getPartitionsList(Hive db) throws HiveException { - if (work.getLoadFileDesc() != null) { - return null; //we are in CTAS, so we know there are no partitions - } - - List list = new ArrayList(); + public static ExecutorService newThreadPool(HiveConf conf) { + int numThreads = HiveConf.getIntVar(conf, ConfVars.HIVE_STATS_GATHER_NUM_THREADS); - if (work.getTableSpecs() != null) { - - // ANALYZE command - TableSpec tblSpec = work.getTableSpecs(); - table = tblSpec.tableHandle; - if (!table.isPartitioned()) { - return null; - } - // get all partitions that matches with the partition spec - List partitions = tblSpec.partitions; - if (partitions != null) { - for (Partition partn : partitions) { - list.add(partn); - } - } - } else if (work.getLoadTableDesc() != null) { - - // INSERT OVERWRITE command - LoadTableDesc tbd = work.getLoadTableDesc(); - table = db.getTable(tbd.getTable().getTableName()); - if (!table.isPartitioned()) { - return null; - } - DynamicPartitionCtx dpCtx = tbd.getDPCtx(); - if (dpCtx != null && dpCtx.getNumDPCols() > 0) { // dynamic partitions - // If no dynamic partitions are generated, dpPartSpecs may not be initialized - if (dpPartSpecs != null) { - // load the list of DP partitions and return the list of partition specs - list.addAll(dpPartSpecs); - } - } else { // static partition - Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); - list.add(partn); - } - } - return list; + ExecutorService executor = Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("StatsNoJobTask-Thread-%d").build()); + LOG.info("Initialized threadpool for stats computation with {} threads", numThreads); + return executor; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java index ab495cfae8..75603ab608 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java @@ -349,6 +349,7 @@ public boolean addDependentTask(Task dependent) { final List> leafTasks = new ArrayList>(); NodeUtils.iterateTask(rootTasks, Task.class, new NodeUtils.Function() { + @Override public void apply(Task task) { List dependents = task.getDependentTasks(); if (dependents == null || dependents.isEmpty()) { @@ -648,4 +649,5 @@ public boolean equals(Object obj) { public boolean canExecuteInParallel(){ return true; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index 36a5eff1e3..e22dc25f8b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.io.merge.MergeFileTask; import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.ConditionalWork; import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -53,8 +53,6 @@ import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.ReplCopyWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.TezWork; /** @@ -98,10 +96,7 @@ public TaskTuple(Class workClass, Class> taskClass) { taskvec.add(new TaskTuple(MapredLocalWork.class, MapredLocalTask.class)); - taskvec.add(new TaskTuple(StatsWork.class, - StatsTask.class)); - taskvec.add(new TaskTuple(StatsNoJobWork.class, StatsNoJobTask.class)); - taskvec.add(new TaskTuple(ColumnStatsWork.class, ColumnStatsTask.class)); + taskvec.add(new TaskTuple(StatsWork.class, StatsTask.class)); taskvec.add(new TaskTuple(ColumnStatsUpdateWork.class, ColumnStatsUpdateTask.class)); taskvec.add(new TaskTuple(MergeFileWork.class, MergeFileTask.class)); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 5214688d24..da13e0a422 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -80,7 +80,6 @@ import org.apache.commons.lang3.StringEscapeUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -119,7 +118,6 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.exec.tez.DagUtils; import org.apache.hadoop.hive.ql.exec.tez.TezTask; -import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -147,11 +145,11 @@ import org.apache.hadoop.hive.ql.metadata.InputEstimator; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; +import org.apache.hadoop.hive.ql.plan.IStatsGatherDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MergeJoinWork; @@ -160,14 +158,12 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.TableDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.api.Adjacency; import org.apache.hadoop.hive.ql.plan.api.Graph; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.stats.StatsFactory; import org.apache.hadoop.hive.ql.stats.StatsPublisher; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; @@ -1456,7 +1452,7 @@ public static void mvFileToFinalPath(Path specPath, Configuration hconf, boolean success, Logger log, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Reporter reporter) throws IOException, HiveException { - + FileSystem fs = specPath.getFileSystem(hconf); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); @@ -1596,12 +1592,14 @@ public static void removeTempOrDuplicateFiles(FileSystem fs, Path path) throws I return removeTempOrDuplicateFiles( fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false, filesKept); } - + private static boolean removeEmptyDpDirectory(FileSystem fs, Path path) throws IOException { FileStatus[] items = fs.listStatus(path); // remove empty directory since DP insert should not generate empty partitions. // empty directories could be generated by crashed Task/ScriptOperator - if (items.length != 0) return false; + if (items.length != 0) { + return false; + } if (!fs.delete(path, true)) { LOG.error("Cannot delete empty directory " + path); throw new IOException("Cannot delete empty directory " + path); @@ -3574,7 +3572,9 @@ private static void createTmpDirs(Configuration conf, if (op instanceof FileSinkOperator) { FileSinkDesc fdesc = ((FileSinkOperator) op).getConf(); - if (fdesc.isMmTable()) continue; // No need to create for MM tables + if (fdesc.isMmTable()) { + continue; // No need to create for MM tables + } Path tempDir = fdesc.getDirName(); if (tempDir != null) { Path tempPath = Utilities.toTempPath(tempDir); @@ -3890,10 +3890,8 @@ public static int getDPColOffset(FileSinkDesc conf) { for (Operator op : ops) { OperatorDesc desc = op.getConf(); String statsTmpDir = null; - if (desc instanceof FileSinkDesc) { - statsTmpDir = ((FileSinkDesc)desc).getStatsTmpDir(); - } else if (desc instanceof TableScanDesc) { - statsTmpDir = ((TableScanDesc) desc).getTmpStatsDir(); + if (desc instanceof IStatsGatherDesc) { + statsTmpDir = ((IStatsGatherDesc) desc).getTmpStatsDir(); } if (statsTmpDir != null && !statsTmpDir.isEmpty()) { statsTmpDirs.add(statsTmpDir); @@ -4063,7 +4061,9 @@ private static boolean isS3(FileSystem fs) { } private static Path[] statusToPath(FileStatus[] statuses) { - if (statuses == null) return null; + if (statuses == null) { + return null; + } Path[] paths = new Path[statuses.length]; for (int i = 0; i < statuses.length; ++i) { paths[i] = statuses[i].getPath(); @@ -4093,7 +4093,9 @@ private static boolean isS3(FileSystem fs) { Utilities.FILE_OP_LOGGER.trace("Looking at " + subDir + " from " + lfs.getPath()); } // If sorted, we'll skip a bunch of files. - if (lastRelDir != null && subDir.startsWith(lastRelDir)) continue; + if (lastRelDir != null && subDir.startsWith(lastRelDir)) { + continue; + } int startIx = skipLevels > 0 ? -1 : 0; for (int i = 0; i < skipLevels; ++i) { startIx = subDir.indexOf(Path.SEPARATOR_CHAR, startIx + 1); @@ -4103,7 +4105,9 @@ private static boolean isS3(FileSystem fs) { break; } } - if (startIx == -1) continue; + if (startIx == -1) { + continue; + } int endIx = subDir.indexOf(Path.SEPARATOR_CHAR, startIx + 1); if (endIx == -1) { Utilities.FILE_OP_LOGGER.info("Expected level of nesting (" + (skipLevels + 1) + ") is not " @@ -4112,7 +4116,9 @@ private static boolean isS3(FileSystem fs) { } lastRelDir = subDir = subDir.substring(0, endIx); Path candidate = new Path(relRoot, subDir); - if (!filter.accept(candidate)) continue; + if (!filter.accept(candidate)) { + continue; + } results.add(fs.makeQualified(candidate)); } return results.toArray(new Path[results.size()]); @@ -4147,7 +4153,9 @@ private static void tryDeleteAllMmFiles(FileSystem fs, Path specPath, Path manif public static void writeMmCommitManifest(List commitPaths, Path specPath, FileSystem fs, String taskId, Long txnId, int stmtId, String unionSuffix) throws HiveException { - if (commitPaths.isEmpty()) return; + if (commitPaths.isEmpty()) { + return; + } // We assume one FSOP per task (per specPath), so we create it in specPath. Path manifestPath = getManifestDir(specPath, txnId, stmtId, unionSuffix); manifestPath = new Path(manifestPath, taskId + MANIFEST_EXTENSION); @@ -4267,11 +4275,15 @@ public static void handleMmTableFinalPath(Path specPath, String unionSuffix, Con throw new HiveException("The following files were committed but not found: " + committed); } - if (mmDirectories.isEmpty()) return; + if (mmDirectories.isEmpty()) { + return; + } // TODO: see HIVE-14886 - removeTempOrDuplicateFiles is broken for list bucketing, // so maintain parity here by not calling it at all. - if (lbLevels != 0) return; + if (lbLevels != 0) { + return; + } // Create fake file statuses to avoid querying the file system. removeTempOrDuplicateFiles // doesn't need tocheck anything except path and directory status for MM directories. FileStatus[] finalResults = new FileStatus[mmDirectories.size()]; @@ -4298,7 +4310,9 @@ private static void cleanMmDirectory(Path dir, FileSystem fs, for (FileStatus child : fs.listStatus(dir)) { Path childPath = child.getPath(); if (unionSuffix == null) { - if (committed.remove(childPath.toString())) continue; // A good file. + if (committed.remove(childPath.toString())) { + continue; // A good file. + } deleteUncommitedFile(childPath, fs); } else if (!child.isDirectory()) { if (committed.contains(childPath.toString())) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index ed9d576fd6..928af6f2a3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1791,7 +1791,9 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par } // column stats will be inaccurate - StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); + if (!hasFollowingStatsTask) { + StatsSetupConst.clearColumnStatsState(newTPart.getParameters()); + } // recreate the partition if it existed before if (isSkewedStoreAsSubdir) { @@ -1810,8 +1812,8 @@ public Partition loadPartition(Path loadPath, Table tbl, Map par if (oldPart == null) { newTPart.getTPartition().setParameters(new HashMap()); if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), null, - StatsSetupConst.TRUE); + StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), + MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE); } MetaStoreUtils.populateQuickStats(HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf)), newTPart.getParameters()); try { @@ -2293,7 +2295,9 @@ public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType } //column stats will be inaccurate - StatsSetupConst.clearColumnStatsState(tbl.getParameters()); + if (!hasFollowingStatsTask) { + StatsSetupConst.clearColumnStatsState(tbl.getParameters()); + } try { if (isSkewedStoreAsSubdir) { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 78e83af4f0..1c262000c8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; @@ -196,7 +197,7 @@ public void truncateTable(String dbName, String tableName, List partName Collections.sort(tableNames); return tableNames; } - + @Override public List getTableMeta(String dbPatterns, String tablePatterns, List tableTypes) throws MetaException { @@ -235,7 +236,7 @@ public void truncateTable(String dbName, String tableName, List partName } return tableMetas; } - + private boolean matchesAny(String string, List matchers) { for (Matcher matcher : matchers) { if (matcher.reset(string).matches()) { @@ -399,6 +400,8 @@ private void createTempTable(org.apache.hadoop.hive.metastore.api.Table tbl, EnvironmentContext envContext) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException { + boolean isVirtualTable = tbl.getTableName().startsWith(SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX); + SessionState ss = SessionState.get(); if (ss == null) { throw new MetaException("No current SessionState, cannot create temporary table" @@ -434,6 +437,10 @@ private void createTempTable(org.apache.hadoop.hive.metastore.api.Table tbl, // Add temp table info to current session Table tTable = new Table(tbl); + if (!isVirtualTable) { + StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), + org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getColumnNamesForTable(tbl), StatsSetupConst.TRUE); + } if (tables == null) { tables = new HashMap(); ss.getTempTables().put(dbName, tables); @@ -466,8 +473,6 @@ private void alterTempTable(String dbname, String tbl_name, } org.apache.hadoop.hive.metastore.api.Table newtCopy = deepCopyAndLowerCaseTable(newt); - MetaStoreUtils.updateTableStatsFast(newtCopy, - getWh().getFileStatusesForSD(newtCopy.getSd()), false, true, envContext); Table newTable = new Table(newtCopy); String newDbName = newTable.getDbName(); String newTableName = newTable.getTableName(); @@ -656,7 +661,7 @@ private void dropTempTable(org.apache.hadoop.hive.metastore.api.Table table, boo public static Map getTempTablesForDatabase(String dbName) { return getTempTables().get(dbName); } - + public static Map> getTempTables() { SessionState ss = SessionState.get(); if (ss == null) { @@ -712,6 +717,13 @@ private boolean updateTempTableColumnStats(String dbName, String tableName, ssTableColStats); } mergeColumnStats(ssTableColStats, colStats); + + List colNames = new ArrayList<>(); + for (ColumnStatisticsObj obj : colStats.getStatsObj()) { + colNames.add(obj.getColName()); + } + org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tableName); + StatsSetupConst.setColumnStatsState(table.getParameters(), colNames); return true; } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java index 768640c544..2ec4b26d64 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java @@ -23,7 +23,6 @@ import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; @@ -41,10 +40,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.mapred.InputFormat; /** @@ -67,8 +66,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, TableScanOperator op = (TableScanOperator) nd; GenMRProcContext ctx = (GenMRProcContext) opProcCtx; ParseContext parseCtx = ctx.getParseCtx(); - Class inputFormat = op.getConf().getTableMetadata() - .getInputFormatClass(); + Table table = op.getConf().getTableMetadata(); + Class inputFormat = table.getInputFormatClass(); Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); // create a dummy MapReduce task @@ -93,19 +92,18 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any MR or Tez job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(op.getConf().getTableMetadata().getTableSpec()); - snjWork.setStatsReliable(parseCtx.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork statWork = new StatsWork(table, parseCtx.getConf()); + statWork.setFooterScan(); + // If partition is specified, get pruned partition list Set confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(op); if (confirmedParts.size() > 0) { - Table source = op.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(op); - PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts, - partCols, false); - snjWork.setPrunedPartitionList(partList); + PrunedPartitionList partList = new PrunedPartitionList(table, confirmedParts, partCols, false); + statWork.addInputPartitions(partList.getPartitions()); + // snjWork.setPrunedPartitionList(partList); } - Task snjTask = TaskFactory.get(snjWork, parseCtx.getConf()); + Task snjTask = TaskFactory.get(statWork, parseCtx.getConf()); ctx.setCurrTask(snjTask); ctx.setCurrTopOp(null); ctx.getRootTasks().clear(); @@ -115,14 +113,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, // The plan consists of a simple MapRedTask followed by a StatsTask. // The MR task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(op.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(op.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(op.getConf().getTmpStatsDir()); - statsWork.setSourceTask(currTask); - statsWork.setStatsReliable(parseCtx.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseCtx.getConf()); - currTask.addDependentTask(statsTask); + BasicStatsWork statsWork = new BasicStatsWork(table.getTableSpec()); + + statsWork.setNoScanAnalyzeCommand(noScan); + StatsWork columnStatsWork = new StatsWork(table, statsWork, parseCtx.getConf()); + columnStatsWork.collectStatsFromAggregator(op.getConf()); + + columnStatsWork.setSourceTask(currTask); + Task columnStatsTask = TaskFactory.get(columnStatsWork, parseCtx.getConf()); + currTask.addDependentTask(columnStatsTask); if (!ctx.getRootTasks().contains(currTask)) { ctx.getRootTasks().add(currTask); } @@ -130,10 +129,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // The plan consists of a StatsTask only. if (noScan) { - statsTask.setParentTasks(null); - statsWork.setNoScanAnalyzeCommand(true); + columnStatsTask.setParentTasks(null); ctx.getRootTasks().remove(currTask); - ctx.getRootTasks().add(statsTask); + ctx.getRootTasks().add(columnStatsTask); } currWork.getMapWork().setGatheringStats(true); @@ -147,9 +145,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, Set confirmedPartns = GenMapRedUtils .getConfirmedPartitionsForScan(op); if (confirmedPartns.size() > 0) { - Table source = op.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(op); - PrunedPartitionList partList = new PrunedPartitionList(source, confirmedPartns, partCols, false); + PrunedPartitionList partList = new PrunedPartitionList(table, confirmedPartns, partCols, false); GenMapRedUtils.setTaskPlan(currAliasId, op, currTask, false, ctx, partList); } else { // non-partitioned table GenMapRedUtils.setTaskPlan(currAliasId, op, currTask, false, ctx); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index dbf4b8da01..7a2601db63 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -34,6 +34,7 @@ import java.util.Set; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.BlobStorageUtils; @@ -85,6 +86,7 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles; import org.apache.hadoop.hive.ql.plan.ConditionalResolverMergeFiles.ConditionalResolverMergeFilesCtx; import org.apache.hadoop.hive.ql.plan.ConditionalWork; @@ -109,7 +111,7 @@ import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.TezWork; @@ -496,6 +498,10 @@ public static void setMapWork(MapWork plan, ParseContext parseCtx, Set currTask, HiveConf hconf) { MoveWork mvWork = mvTask.getWork(); - StatsWork statsWork = null; + BasicStatsWork statsWork = null; + Table table = null; + boolean truncate = false; if (mvWork.getLoadTableWork() != null) { - statsWork = new StatsWork(mvWork.getLoadTableWork()); + statsWork = new BasicStatsWork(mvWork.getLoadTableWork()); + String tableName = mvWork.getLoadTableWork().getTable().getTableName(); + truncate = mvWork.getLoadTableWork().getReplace(); + try { + table = Hive.get().getTable(SessionState.get().getCurrentDatabase(), tableName); + } catch (HiveException e) { + throw new RuntimeException("unexpected; table should be present already..: " + tableName, e); + } } else if (mvWork.getLoadFileWork() != null) { - statsWork = new StatsWork(mvWork.getLoadFileWork()); + statsWork = new BasicStatsWork(mvWork.getLoadFileWork()); + + truncate = true; + if (mvWork.getLoadFileWork().getCtasCreateTableDesc() == null) { + throw new RuntimeException("unexpected; this should be a CTAS - however no desc present"); + } + try { + table = mvWork.getLoadFileWork().getCtasCreateTableDesc().toTable(hconf); + } catch (HiveException e) { + LOG.debug("can pre-create table", e); + table = null; + } } assert statsWork != null : "Error when generating StatsTask"; - statsWork.setSourceTask(currTask); - statsWork.setStatsReliable(hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE)); - statsWork.setStatsTmpDir(nd.getConf().getStatsTmpDir()); if (currTask.getWork() instanceof MapredWork) { MapredWork mrWork = (MapredWork) currTask.getWork(); mrWork.getMapWork().setGatheringStats(true); @@ -1509,10 +1532,15 @@ public static void addStatsTask(FileSinkOperator nd, MoveTask mvTask, } } - // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix - // in FileSinkDesc is used for stats publishing. They should be consistent. - statsWork.setAggKey(nd.getConf().getStatsAggPrefix()); - Task statsTask = TaskFactory.get(statsWork, hconf); + StatsWork columnStatsWork = new StatsWork(table, statsWork, hconf); + columnStatsWork.collectStatsFromAggregator(nd.getConf()); + + columnStatsWork.truncateExisting(truncate); + // columnStatsWork.collectStatsFromAggregator(nd.getConf()); + + + columnStatsWork.setSourceTask(currTask); + Task statsTask = TaskFactory.get(columnStatsWork, hconf); // subscribe feeds from the MoveTask so that MoveTask can forward the list // of dynamic partition list to the StatsTask diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java index 91c6c0050b..624eb6ced2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hive.ql.optimizer; +import java.util.List; +import java.util.Set; + import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorUtils; @@ -30,8 +33,9 @@ import org.apache.hadoop.hive.ql.parse.spark.GenSparkWork; import org.apache.hadoop.hive.ql.plan.ArchiveWork; import org.apache.hadoop.hive.ql.plan.BaseWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; import org.apache.hadoop.hive.ql.plan.ConditionalWork; import org.apache.hadoop.hive.ql.plan.CopyWork; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -45,15 +49,11 @@ import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.TezWork; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.Set; - /** * Finds Acid FileSinkDesc objects which can be created in the physical (disconnected) plan, e.g. * {@link org.apache.hadoop.hive.ql.parse.GenTezUtils#removeUnionOperators(GenTezProcContext, BaseWork, int)} @@ -92,7 +92,7 @@ else if(work instanceof SparkWork) { } else if(work instanceof MapredLocalWork) { //I don't think this can have any FileSinkOperatorS - more future proofing - Set fileSinkOperatorSet = OperatorUtils.findOperators(((MapredLocalWork)work).getAliasToWork().values(), FileSinkOperator.class); + Set fileSinkOperatorSet = OperatorUtils.findOperators(((MapredLocalWork) work).getAliasToWork().values(), FileSinkOperator.class); for(FileSinkOperator fsop : fileSinkOperatorSet) { collectFileSinkDescs(fsop, acidSinks); } @@ -100,42 +100,6 @@ else if(work instanceof MapredLocalWork) { else if(work instanceof ExplainWork) { new QueryPlanPostProcessor(((ExplainWork)work).getRootTasks(), acidSinks, executionId); } - /* - ekoifman:~ ekoifman$ cd dev/hiverwgit/ql/src/java/org/apache/ -ekoifman:apache ekoifman$ find . -name *Work.java -./hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java -./hadoop/hive/ql/exec/repl/ReplDumpWork.java -./hadoop/hive/ql/exec/repl/ReplStateLogWork.java -./hadoop/hive/ql/index/IndexMetadataChangeWork.java -./hadoop/hive/ql/io/merge/MergeFileWork.java - extends MapWork -./hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java - extends MapWork -./hadoop/hive/ql/parse/GenTezWork.java -./hadoop/hive/ql/parse/spark/GenSparkWork.java -./hadoop/hive/ql/plan/ArchiveWork.java -./hadoop/hive/ql/plan/BaseWork.java -./hadoop/hive/ql/plan/ColumnStatsUpdateWork.java -./hadoop/hive/ql/plan/ColumnStatsWork.java -./hadoop/hive/ql/plan/ConditionalWork.java -./hadoop/hive/ql/plan/CopyWork.java -./hadoop/hive/ql/plan/DDLWork.java -./hadoop/hive/ql/plan/DependencyCollectionWork.java -./hadoop/hive/ql/plan/ExplainSQRewriteWork.java -./hadoop/hive/ql/plan/ExplainWork.java -./hadoop/hive/ql/plan/FetchWork.java -./hadoop/hive/ql/plan/FunctionWork.java -./hadoop/hive/ql/plan/MapredLocalWork.java -./hadoop/hive/ql/plan/MapredWork.java -./hadoop/hive/ql/plan/MapWork.java - extends BaseWork -./hadoop/hive/ql/plan/MergeJoinWork.java - extends BaseWork -./hadoop/hive/ql/plan/MoveWork.java -./hadoop/hive/ql/plan/ReduceWork.java -./hadoop/hive/ql/plan/ReplCopyWork.java - extends CopyWork -./hadoop/hive/ql/plan/SparkWork.java -./hadoop/hive/ql/plan/StatsNoJobWork.java -./hadoop/hive/ql/plan/StatsWork.java -./hadoop/hive/ql/plan/TezWork.java -./hadoop/hive/ql/plan/UnionWork.java - extends BaseWork - */ else if(work instanceof ReplLoadWork || work instanceof ReplStateLogWork || work instanceof IndexMetadataChangeWork || @@ -143,7 +107,7 @@ else if(work instanceof ReplLoadWork || work instanceof GenSparkWork || work instanceof ArchiveWork || work instanceof ColumnStatsUpdateWork || - work instanceof ColumnStatsWork || + work instanceof BasicStatsWork || work instanceof ConditionalWork || work instanceof CopyWork || work instanceof DDLWork || @@ -152,7 +116,7 @@ else if(work instanceof ReplLoadWork || work instanceof FetchWork || work instanceof FunctionWork || work instanceof MoveWork || - work instanceof StatsNoJobWork || + work instanceof BasicStatsNoJobWork || work instanceof StatsWork) { LOG.debug("Found " + work.getClass().getName() + " - no FileSinkOperation can be present. executionId=" + executionId); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java index 3a20cfe7ac..bf7a6447df 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java @@ -22,11 +22,9 @@ import java.util.Comparator; import java.util.Iterator; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; import java.util.SortedSet; import java.util.Stack; @@ -35,12 +33,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.exec.tez.DagUtils; import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; @@ -58,8 +53,6 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MergeJoinWork; import org.apache.hadoop.hive.ql.plan.ReduceWork; -import org.apache.hadoop.hive.ql.plan.TezEdgeProperty; -import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType; import org.apache.hadoop.hive.ql.plan.TezWork; /** @@ -174,6 +167,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } Comparator comp = new Comparator() { + @Override public int compare(MapJoinOperator mj1, MapJoinOperator mj2) { if (mj1 == null || mj2 == null) { throw new NullPointerException(); diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java index dc433fed22..6a0ca5dc3c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SerializeFilter.java @@ -26,8 +26,8 @@ import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.tez.TezTask; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java index 2f9783ed18..e1df60ba09 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java @@ -86,7 +86,9 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) ParseContext pc = physicalContext.getParseContext(); if (pc.getLoadTableWork() != null) { for (LoadTableDesc ltd : pc.getLoadTableWork()) { - if (!ltd.isMmTable()) continue; + if (!ltd.isMmTable()) { + continue; + } // See the path in FSOP that calls fs.exists on finalPath. LOG.debug("Not using skew join because the destination table " + ltd.getTable().getTableName() + " is an insert_only table"); @@ -95,9 +97,10 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) } if (pc.getLoadFileWork() != null) { for (LoadFileDesc lfd : pc.getLoadFileWork()) { - if (!lfd.isMmCtas()) continue; - LOG.debug("Not using skew join because the destination table " - + lfd.getDestinationCreateTable() + " is an insert_only table"); + if (!lfd.isMmCtas()) { + continue; + } + LOG.debug("Not using skew join because the destination table is an insert_only table"); return null; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 840e4cecd3..f4b410efd7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -81,7 +81,6 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.hive.ql.stats.StatsUtils; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -368,10 +367,10 @@ public static String getUnescapedName(ASTNode tableOrColumnNode, String currentD String dbName = dbTablePair.getKey(); String tableName = dbTablePair.getValue(); if (dbName != null){ - return StatsUtils.getFullyQualifiedTableName(dbName, tableName); + return dbName + "." + tableName; } if (currentDatabase != null) { - return StatsUtils.getFullyQualifiedTableName(currentDatabase, tableName); + return currentDatabase + "." + tableName; } return tableName; } else if (tokenType == HiveParser.StringLiteral) { @@ -794,8 +793,8 @@ private static void generateConstraintInfos(ASTNode child, List columnNa } if (enable) { throw new SemanticException( - ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("ENABLE/ENFORCED feature not supported yet. " - + "Please use DISABLE/NOT ENFORCED instead.")); + ErrorMsg.INVALID_CSTR_SYNTAX.getMsg("ENABLE feature not supported yet. " + + "Please use DISABLE instead.")); } if (validate) { throw new SemanticException( @@ -1119,19 +1118,22 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast) public TableSpec(Table table) { tableHandle = table; - tableName = table.getFullyQualifiedName(); + tableName = table.getDbName() + "." + table.getTableName(); specType = SpecType.TABLE_ONLY; } public TableSpec(Hive db, String tableName, Map partSpec) throws HiveException { Table table = db.getTable(tableName); - final Partition partition = partSpec == null ? null : db.getPartition(table, partSpec, false); tableHandle = table; - this.tableName = table.getFullyQualifiedName(); - if (partition == null) { + this.tableName = table.getDbName() + "." + table.getTableName(); + if (partSpec == null) { specType = SpecType.TABLE_ONLY; } else { + Partition partition = db.getPartition(table, partSpec, false); + if (partition == null) { + throw new SemanticException("partition is unknown: " + table + "/" + partSpec); + } partHandle = partition; partitions = Collections.singletonList(partHandle); specType = SpecType.STATIC_PARTITION; @@ -1732,7 +1734,9 @@ public static void validatePartColumnType(Table tbl, Map partSpe @VisibleForTesting static void normalizeColSpec(Map partSpec, String colName, String colType, String originalColSpec, Object colValue) throws SemanticException { - if (colValue == null) return; // nothing to do with nulls + if (colValue == null) { + return; // nothing to do with nulls + } String normalizedColSpec = originalColSpec; if (colType.equals(serdeConstants.DATE_TYPE_NAME)) { normalizedColSpec = normalizeDateCol(colValue, originalColSpec); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index dc29b122c9..de7bde73fc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc; import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.CreateResourcePlanDesc; @@ -137,7 +138,7 @@ import org.apache.hadoop.hive.ql.plan.ShowTablesDesc; import org.apache.hadoop.hive.ql.plan.ShowTblPropertiesDesc; import org.apache.hadoop.hive.ql.plan.ShowTxnsDesc; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TruncateTableDesc; @@ -1208,18 +1209,19 @@ private void analyzeTruncateTable(ASTNode ast) throws SemanticException { // Recalculate the HDFS stats if auto gather stats is set if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc; + BasicStatsWork basicStatsWork; if (oldTblPartLoc.equals(newTblPartLoc)) { // If we're merging to the same location, we can avoid some metastore calls TableSpec tablepart = new TableSpec(this.db, conf, root); - statDesc = new StatsWork(tablepart); + basicStatsWork = new BasicStatsWork(tablepart); } else { - statDesc = new StatsWork(ltd); + basicStatsWork = new BasicStatsWork(ltd); } - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statTask = TaskFactory.get(statDesc, conf); + basicStatsWork.setNoStatsAggregator(true); + basicStatsWork.setClearAggregatorStats(true); + StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, conf); + + Task statTask = TaskFactory.get(columnStatsWork, conf); moveTsk.addDependentTask(statTask); } } catch (HiveException e) { @@ -1867,18 +1869,36 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, mergeTask.addDependentTask(moveTsk); if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc; + BasicStatsWork basicStatsWork; if (oldTblPartLoc.equals(newTblPartLoc)) { // If we're merging to the same location, we can avoid some metastore calls TableSpec tableSpec = new TableSpec(db, tableName, partSpec); - statDesc = new StatsWork(tableSpec); + basicStatsWork = new BasicStatsWork(tableSpec); } else { - statDesc = new StatsWork(ltd); + basicStatsWork = new BasicStatsWork(ltd); } - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statTask = TaskFactory.get(statDesc, conf); + basicStatsWork.setNoStatsAggregator(true); + basicStatsWork.setClearAggregatorStats(true); + StatsWork columnStatsWork = new StatsWork(tblObj, basicStatsWork, conf); + + //////// BasicStatsWork basicStatsWork; + // if (oldTblPartLoc.equals(newTblPartLoc)) { + // // If we're merging to the same location, we can avoid some metastore calls + // TableSpec tableSpec = new TableSpec(db, tableName, partSpec); + // basicStatsWork = new BasicStatsWork(tableSpec); + // } else { + // basicStatsWork = new BasicStatsWork(ltd); + // } + // + // basicStatsWork.setNoStatsAggregator(true); + // basicStatsWork.setClearAggregatorStats(true); + // // StatsWork columnStatsWork2 = new StatsWork(tblObj, basicStatsWork, conf); + // // columnStatsWork2.addPartition(partSpec); + // // + // // columnStatsWork2.tru + + + Task statTask = TaskFactory.get(columnStatsWork, conf); moveTsk.addDependentTask(statTask); } @@ -1980,7 +2000,7 @@ private void analyzeAlterTableAddConstraint(ASTNode ast, String tableName) switch (child.getToken().getType()) { case HiveParser.TOK_UNIQUE: BaseSemanticAnalyzer.processUniqueConstraints(qualifiedTabName[0], qualifiedTabName[1], - child, uniqueConstraints); + child, uniqueConstraints); break; case HiveParser.TOK_PRIMARY_KEY: BaseSemanticAnalyzer.processPrimaryKeys(qualifiedTabName[0], qualifiedTabName[1], diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java index 7a0d4a752e..065c7e5098 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; -import org.antlr.runtime.TokenRewriteStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,17 +39,12 @@ import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.ExplainTask; import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.VectorizationDetailLevel; import org.apache.hadoop.hive.ql.plan.ExplainWork; -import org.apache.hadoop.hive.ql.processors.CommandProcessor; -import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.stats.StatsAggregator; import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; import org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 01cb2b31a5..b6f1139fe1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -205,10 +205,6 @@ protected void setupMapWork(MapWork mapWork, GenTezProcContext context, // All the setup is done in GenMapRedUtils GenMapRedUtils.setMapWork(mapWork, context.parseContext, context.inputs, partitions, root, alias, context.conf, false); - // we also collect table stats while collecting column stats. - if (context.parseContext.getAnalyzeRewrite() != null) { - mapWork.setGatheringStats(true); - } } // removes any union operator and clones the plan diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 5f2a34ef57..238fbd6057 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; - import java.io.IOException; import java.io.Serializable; import java.net.URI; @@ -36,7 +35,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -50,11 +48,12 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; import org.apache.hadoop.hive.ql.plan.MoveWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.InputFormat; @@ -225,8 +224,10 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { List bucketCols = ts.tableHandle.getBucketCols(); if (bucketCols != null && !bucketCols.isEmpty()) { String error = StrictChecks.checkBucketing(conf); - if (error != null) throw new SemanticException("Please load into an intermediate table" - + " and use 'insert... select' to allow Hive to enforce bucketing. " + error); + if (error != null) { + throw new SemanticException("Please load into an intermediate table" + + " and use 'insert... select' to allow Hive to enforce bucketing. " + error); + } } if(AcidUtils.isAcidTable(ts.tableHandle) && !AcidUtils.isInsertOnlyTable(ts.tableHandle.getParameters())) { @@ -313,11 +314,11 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { // Update the stats which do not require a complete scan. Task statTask = null; if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { - StatsWork statDesc = new StatsWork(loadTableWork); - statDesc.setNoStatsAggregator(true); - statDesc.setClearAggregatorStats(true); - statDesc.setStatsReliable(conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - statTask = TaskFactory.get(statDesc, conf); + BasicStatsWork basicStatsWork = new BasicStatsWork(loadTableWork); + basicStatsWork.setNoStatsAggregator(true); + basicStatsWork.setClearAggregatorStats(true); + StatsWork columnStatsWork = new StatsWork(ts.tableHandle, basicStatsWork, conf); + statTask = TaskFactory.get(columnStatsWork, conf); } // HIVE-3334 has been filed for load file with index auto update diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java index 9309fbd0e3..1c2ad7b6c8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.parse; -import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.Stack; @@ -26,30 +25,25 @@ import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; -import org.apache.hadoop.hive.ql.plan.MapWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.MapWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.mapred.InputFormat; /** * ProcessAnalyzeTable sets up work for the several variants of analyze table - * (normal, no scan, partial scan.) The plan at this point will be a single + * (normal, no scan.) The plan at this point will be a single * table scan operator. */ public class ProcessAnalyzeTable implements NodeProcessor { @@ -76,8 +70,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, TableScanOperator tableScan = (TableScanOperator) nd; ParseContext parseContext = context.parseContext; - Class inputFormat = tableScan.getConf().getTableMetadata() - .getInputFormatClass(); + Table table = tableScan.getConf().getTableMetadata(); + Class inputFormat = table.getInputFormatClass(); if (parseContext.getQueryProperties().isAnalyzeCommand()) { @@ -99,20 +93,17 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any Tez job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata() - .getTableSpec()); - snjWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork statWork = new StatsWork(table, parseContext.getConf()); + statWork.setFooterScan(); + // If partition is specified, get pruned partition list Set confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan); if (confirmedParts.size() > 0) { - Table source = tableScan.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(tableScan); - PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts, partCols, - false); - snjWork.setPrunedPartitionList(partList); + PrunedPartitionList partList = new PrunedPartitionList(table, confirmedParts, partCols, false); + statWork.addInputPartitions(partList.getPartitions()); } - Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); + Task snjTask = TaskFactory.get(statWork, parseContext.getConf()); snjTask.setParentTasks(null); context.rootTasks.remove(context.currentTask); context.rootTasks.add(snjTask); @@ -123,20 +114,19 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, // The plan consists of a simple TezTask followed by a StatsTask. // The Tez task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseContext.getConf()); + BasicStatsWork basicStatsWork = new BasicStatsWork(table.getTableSpec()); + basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand()); + StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, parseContext.getConf()); + columnStatsWork.collectStatsFromAggregator(tableScan.getConf()); + + columnStatsWork.setSourceTask(context.currentTask); + Task statsTask = TaskFactory.get(columnStatsWork, parseContext.getConf()); context.currentTask.addDependentTask(statsTask); // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // The plan consists of a StatsTask only. if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) { statsTask.setParentTasks(null); - statsWork.setNoScanAnalyzeCommand(true); context.rootTasks.remove(context.currentTask); context.rootTasks.add(statsTask); } @@ -147,9 +137,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, Set confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan); PrunedPartitionList partitions = null; if (confirmedPartns.size() > 0) { - Table source = tableScan.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(tableScan); - partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false); + partitions = new PrunedPartitionList(table, confirmedPartns, partCols, false); } MapWork w = utils.createMapWork(context, tableScan, tezWork, partitions); @@ -157,55 +146,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, return true; } - } else if (parseContext.getAnalyzeRewrite() != null) { - // we need to collect table stats while collecting column stats. - try { - context.currentTask.addDependentTask(genTableStats(context, tableScan)); - } catch (HiveException e) { - throw new SemanticException(e); - } } return null; } - - private Task genTableStats(GenTezProcContext context, TableScanOperator tableScan) - throws HiveException { - Class inputFormat = tableScan.getConf().getTableMetadata() - .getInputFormatClass(); - ParseContext parseContext = context.parseContext; - Table table = tableScan.getConf().getTableMetadata(); - List partitions = new ArrayList<>(); - if (table.isPartitioned()) { - partitions.addAll(parseContext.getPrunedPartitions(tableScan).getPartitions()); - for (Partition partn : partitions) { - LOG.debug("XXX: adding part: " + partn); - context.outputs.add(new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK)); - } - } - TableSpec tableSpec = new TableSpec(table, partitions); - tableScan.getConf().getTableMetadata().setTableSpec(tableSpec); - - if (inputFormat.equals(OrcInputFormat.class)) { - // For ORC, there is no Tez Job for table stats. - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata() - .getTableSpec()); - snjWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - // If partition is specified, get pruned partition list - if (partitions.size() > 0) { - snjWork.setPrunedPartitionList(parseContext.getPrunedPartitions(tableScan)); - } - return TaskFactory.get(snjWork, parseContext.getConf()); - } else { - - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - return TaskFactory.get(statsWork, parseContext.getConf()); - } - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5ccb69ac48..e22c11baed 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -6997,7 +6997,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) Utilities.FILE_OP_LOGGER.trace("Setting query directory " + queryTmpdir + " from " + dest_path + " (" + isMmTable + ")"); } - } catch (Exception e) { + } catch (Exception e) { throw new SemanticException("Error creating temporary folder on: " + dest_path, e); } @@ -7145,7 +7145,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // the following code is used to collect column stats when // hive.stats.autogather=true // and it is an insert overwrite or insert into table - if (dest_tab != null && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) + if (dest_tab != null && !dest_tab.isNonNative() + && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER) && ColumnStatsAutoGatherContext.canRunAutogatherStats(fso)) { if (dest_type.intValue() == QBMetaData.DEST_TABLE) { @@ -7405,7 +7406,9 @@ private void checkImmutableTable(QB qb, Table dest_tab, Path dest_path, boolean private DynamicPartitionCtx checkDynPart(QB qb, QBMetaData qbm, Table dest_tab, Map partSpec, String dest) throws SemanticException { List parts = dest_tab.getPartitionKeys(); - if (parts == null || parts.isEmpty()) return null; // table is not partitioned + if (parts == null || parts.isEmpty()) { + return null; // table is not partitioned + } if (partSpec == null || partSpec.size() == 0) { // user did NOT specify partition throw new SemanticException(generateErrorMessage(qb.getParseInfo().getDestForClause(dest), ErrorMsg.NEED_PARTITION_ERROR.getMsg())); @@ -10566,10 +10569,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String throws SemanticException { // if it is not analyze command and not column stats, then do not gatherstats - // if it is column stats, but it is not tez, do not gatherstats - if ((!qbp.isAnalyzeCommand() && qbp.getAnalyzeRewrite() == null) - || (qbp.getAnalyzeRewrite() != null && !HiveConf.getVar(conf, - HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))) { + if (!qbp.isAnalyzeCommand() && qbp.getAnalyzeRewrite() == null) { tsDesc.setGatherStats(false); } else { if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { @@ -13644,9 +13644,12 @@ private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo, // Don't know the characteristics of non-native tables, // and don't have a rational way to guess, so assume the most // conservative case. - if (isNonNativeTable) return WriteEntity.WriteType.INSERT_OVERWRITE; - else return ((ltd.getLoadFileType() == LoadFileType.REPLACE_ALL) - ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest)); + if (isNonNativeTable) { + return WriteEntity.WriteType.INSERT_OVERWRITE; + } else { + return ((ltd.getLoadFileType() == LoadFileType.REPLACE_ALL) + ? WriteEntity.WriteType.INSERT_OVERWRITE : getWriteType(dest)); + } } private WriteEntity.WriteType getWriteType(String dest) { diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index a63f709eef..406fe2c534 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -22,8 +22,10 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -39,9 +41,9 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.exec.ColumnStatsTask; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -49,12 +51,16 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; import org.apache.hadoop.hive.ql.plan.CreateViewDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -64,6 +70,7 @@ import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; @@ -74,16 +81,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.thrift.ThriftFormatter; import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; +import org.apache.hadoop.mapred.InputFormat; /** * TaskCompiler is a the base class for classes that compile @@ -273,9 +271,65 @@ public void compile(final ParseContext pCtx, final List map = new LinkedHashMap<>(); + if (isCStats) { + if (rootTasks == null || rootTasks.size() != 1 || pCtx.getTopOps() == null + || pCtx.getTopOps().size() != 1) { + throw new SemanticException("Can not find correct root task!"); + } + try { + Task root = rootTasks.iterator().next(); + StatsTask tsk = (StatsTask) genTableStats(pCtx, pCtx.getTopOps().values() + .iterator().next(), root, outputs); + root.addDependentTask(tsk); + map.put(extractTableFullName(tsk), tsk); + } catch (HiveException e) { + throw new SemanticException(e); + } + genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadFileWork, map, outerQueryLimit, 0); + } else { + Set> leafTasks = new LinkedHashSet>(); + getLeafTasks(rootTasks, leafTasks); + List> nonStatsLeafTasks = new ArrayList<>(); + for (Task tsk : leafTasks) { + // map table name to the correct ColumnStatsTask + if (tsk instanceof StatsTask) { + map.put(extractTableFullName((StatsTask) tsk), (StatsTask) tsk); + } else { + nonStatsLeafTasks.add(tsk); + } + } + // add cStatsTask as a dependent of all the nonStatsLeafTasks + for (Task tsk : nonStatsLeafTasks) { + for (Task cStatsTask : map.values()) { + tsk.addDependentTask(cStatsTask); + } + } + for (ColumnStatsAutoGatherContext columnStatsAutoGatherContext : pCtx + .getColumnStatsAutoGatherContexts()) { + if (!columnStatsAutoGatherContext.isInsertInto()) { + genColumnStatsTask(columnStatsAutoGatherContext.getAnalyzeRewrite(), + columnStatsAutoGatherContext.getLoadFileWork(), map, outerQueryLimit, 0); + } else { + int numBitVector; + try { + numBitVector = HiveStatsUtils.getNumBitVectorsForNDVEstimation(conf); + } catch (Exception e) { + throw new SemanticException(e.getMessage()); + } + genColumnStatsTask(columnStatsAutoGatherContext.getAnalyzeRewrite(), + columnStatsAutoGatherContext.getLoadFileWork(), map, outerQueryLimit, numBitVector); + } + } + } } decideExecMode(rootTasks, ctx, globalLimitCtx); @@ -322,6 +376,45 @@ public void compile(final ParseContext pCtx, final List genTableStats(ParseContext parseContext, TableScanOperator tableScan, Task currentTask, final HashSet outputs) + throws HiveException { + Class inputFormat = tableScan.getConf().getTableMetadata() + .getInputFormatClass(); + Table table = tableScan.getConf().getTableMetadata(); + List partitions = new ArrayList<>(); + if (table.isPartitioned()) { + partitions.addAll(parseContext.getPrunedPartitions(tableScan).getPartitions()); + for (Partition partn : partitions) { + LOG.debug("XXX: adding part: " + partn); + outputs.add(new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK)); + } + } + TableSpec tableSpec = new TableSpec(table, partitions); + tableScan.getConf().getTableMetadata().setTableSpec(tableSpec); + + if (inputFormat.equals(OrcInputFormat.class)) { + // For ORC, there is no Tez Job for table stats. + StatsWork columnStatsWork = new StatsWork(table, parseContext.getConf()); + columnStatsWork.setFooterScan(); + // If partition is specified, get pruned partition list + if (partitions.size() > 0) { + columnStatsWork.addInputPartitions(parseContext.getPrunedPartitions(tableScan).getPartitions()); + // snjWork.setPrunedPartitionList(parseContext.getPrunedPartitions(tableScan)); + } + return TaskFactory.get(columnStatsWork, parseContext.getConf()); + } else { + BasicStatsWork statsWork = new BasicStatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + StatsWork columnStatsWork = new StatsWork(table, statsWork, parseContext.getConf()); + columnStatsWork.collectStatsFromAggregator(tableScan.getConf()); + columnStatsWork.setSourceTask(currentTask); + return TaskFactory.get(columnStatsWork, parseContext.getConf()); + } + } + private void setLoadFileLocation( final ParseContext pCtx, LoadFileDesc lfd) throws SemanticException { // CTAS; make the movetask's destination directory the table's destination. @@ -353,34 +446,6 @@ private void setLoadFileLocation( lfd.setTargetDir(location); } - private void createColumnStatsTasks(final ParseContext pCtx, - final List> rootTasks, - List loadFileWork, boolean isCStats, int outerQueryLimit) - throws SemanticException { - Set> leafTasks = new LinkedHashSet>(); - getLeafTasks(rootTasks, leafTasks); - if (isCStats) { - genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadFileWork, leafTasks, outerQueryLimit, 0); - } else { - for (ColumnStatsAutoGatherContext columnStatsAutoGatherContext : pCtx - .getColumnStatsAutoGatherContexts()) { - if (!columnStatsAutoGatherContext.isInsertInto()) { - genColumnStatsTask(columnStatsAutoGatherContext.getAnalyzeRewrite(), - columnStatsAutoGatherContext.getLoadFileWork(), leafTasks, outerQueryLimit, 0); - } else { - int numBitVector; - try { - numBitVector = HiveStatsUtils.getNumBitVectorsForNDVEstimation(conf); - } catch (Exception e) { - throw new SemanticException(e.getMessage()); - } - genColumnStatsTask(columnStatsAutoGatherContext.getAnalyzeRewrite(), - columnStatsAutoGatherContext.getLoadFileWork(), leafTasks, outerQueryLimit, numBitVector); - } - } - } - } - private Path getDefaultCtasLocation(final ParseContext pCtx) throws SemanticException { try { String protoName = null; @@ -419,10 +484,8 @@ private void patchUpAfterCTASorMaterializedView(final List> leaves = - new LinkedHashSet<>(); + // find all leaf tasks and make the DDLTask as a dependent task on all of them + HashSet> leaves = new LinkedHashSet<>(); getLeafTasks(rootTasks, leaves); assert (leaves.size() > 0); for (Task task : leaves) { @@ -452,10 +515,8 @@ private void patchUpAfterCTASorMaterializedView(final List loadFileWork, Set> leafTasks, - int outerQueryLimit, int numBitVector) { - ColumnStatsTask cStatsTask; - ColumnStatsWork cStatsWork; + List loadFileWork, Map map, + int outerQueryLimit, int numBitVector) throws SemanticException { FetchWork fetch; String tableName = analyzeRewrite.getTableName(); List colName = analyzeRewrite.getColName(); @@ -482,11 +543,12 @@ protected void genColumnStatsTask(AnalyzeRewriteContext analyzeRewrite, fetch = new FetchWork(loadFileWork.get(0).getSourcePath(), resultTab, outerQueryLimit); ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName, - colName, colType, isTblLevel, numBitVector); - cStatsWork = new ColumnStatsWork(fetch, cStatsDesc, SessionState.get().getCurrentDatabase()); - cStatsTask = (ColumnStatsTask) TaskFactory.get(cStatsWork, conf); - for (Task tsk : leafTasks) { - tsk.addDependentTask(cStatsTask); + colName, colType, isTblLevel, numBitVector, fetch); + StatsTask columnStatsTask = map.get(tableName); + if (columnStatsTask == null) { + throw new SemanticException("Can not find " + tableName + " in genColumnStatsTask"); + } else { + columnStatsTask.getWork().setColStats(cStatsDesc); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java index 6f21cae8d6..36fe8a24b4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -40,10 +39,10 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; -import org.apache.hadoop.hive.ql.plan.StatsWork; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.mapred.InputFormat; import com.google.common.base.Preconditions; @@ -78,9 +77,9 @@ public Object process(Node nd, Stack stack, ParseContext parseContext = context.parseContext; + Table table = tableScan.getConf().getTableMetadata(); @SuppressWarnings("rawtypes") - Class inputFormat = tableScan.getConf().getTableMetadata() - .getInputFormatClass(); + Class inputFormat = table.getInputFormatClass(); if (parseContext.getQueryProperties().isAnalyzeCommand()) { Preconditions.checkArgument(tableScan.getChildOperators() == null @@ -103,19 +102,16 @@ public Object process(Node nd, Stack stack, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any Spark job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec()); - snjWork.setStatsReliable(parseContext.getConf().getBoolVar( - HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + StatsWork statWork = new StatsWork(table, parseContext.getConf()); + statWork.setFooterScan(); // If partition is specified, get pruned partition list Set confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan); if (confirmedParts.size() > 0) { - Table source = tableScan.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(tableScan); - PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts, partCols, - false); - snjWork.setPrunedPartitionList(partList); + PrunedPartitionList partList = new PrunedPartitionList(table, confirmedParts, partCols, false); + statWork.addInputPartitions(partList.getPartitions()); } - Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); + Task snjTask = TaskFactory.get(statWork, parseContext.getConf()); snjTask.setParentTasks(null); context.rootTasks.remove(context.currentTask); context.rootTasks.add(snjTask); @@ -126,19 +122,18 @@ public Object process(Node nd, Stack stack, // The plan consists of a simple SparkTask followed by a StatsTask. // The Spark task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseContext.getConf()); + BasicStatsWork basicStatsWork = new BasicStatsWork(table.getTableSpec()); + basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand()); + StatsWork columnStatsWork = new StatsWork(table, basicStatsWork, parseContext.getConf()); + columnStatsWork.collectStatsFromAggregator(tableScan.getConf()); + columnStatsWork.setSourceTask(context.currentTask); + Task statsTask = TaskFactory.get(columnStatsWork, parseContext.getConf()); context.currentTask.addDependentTask(statsTask); // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // The plan consists of a StatsTask only. if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) { statsTask.setParentTasks(null); - statsWork.setNoScanAnalyzeCommand(true); context.rootTasks.remove(context.currentTask); context.rootTasks.add(statsTask); } @@ -148,9 +143,8 @@ public Object process(Node nd, Stack stack, Set confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan); PrunedPartitionList partitions = null; if (confirmedPartns.size() > 0) { - Table source = tableScan.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(tableScan); - partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false); + partitions = new PrunedPartitionList(table, confirmedPartns, partCols, false); } MapWork w = utils.createMapWork(context, tableScan, sparkWork, partitions); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsNoJobWork.java similarity index 61% rename from ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java rename to ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsNoJobWork.java index 77c04f6c6e..97d692dcba 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsNoJobWork.java @@ -19,52 +19,48 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.Set; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; -import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; -import org.apache.hadoop.hive.ql.plan.Explain.Level; - - /** * Client-side stats aggregator task. */ -@Explain(displayName = "Stats-Aggr Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class StatsNoJobWork implements Serializable { +public class BasicStatsNoJobWork implements Serializable { private static final long serialVersionUID = 1L; private TableSpec tableSpecs; private boolean statsReliable; - private PrunedPartitionList prunedPartitionList; + // private PrunedPartitionList prunedPartitionList; - public StatsNoJobWork() { - } + private Set pp; - public StatsNoJobWork(TableSpec tableSpecs) { + public + //public + BasicStatsNoJobWork(TableSpec tableSpecs) { this.tableSpecs = tableSpecs; } - public StatsNoJobWork(boolean statsReliable) { - this.statsReliable = statsReliable; - } - public TableSpec getTableSpecs() { return tableSpecs; } + public void setStatsReliable(boolean s1) { + statsReliable = s1; + } + public boolean isStatsReliable() { return statsReliable; } - public void setStatsReliable(boolean statsReliable) { - this.statsReliable = statsReliable; + public Set getPartitions() { + return pp; } - public void setPrunedPartitionList(PrunedPartitionList prunedPartitionList) { - this.prunedPartitionList = prunedPartitionList; - } + public void setPartitions(Set partitions) { + pp=partitions; - public PrunedPartitionList getPrunedPartitionList() { - return prunedPartitionList; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java new file mode 100644 index 0000000000..0621bd413e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BasicStatsWork.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +import java.io.Serializable; + +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.plan.Explain.Level; +import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; + + +/** + * ConditionalStats. + * + */ +public class BasicStatsWork implements Serializable { + private static final long serialVersionUID = 1L; + + private TableSpec tableSpecs; // source table spec -- for TableScanOperator + private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator + private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator + private String aggKey; // aggregation key prefix + private boolean statsReliable; // are stats completely reliable + + // If stats aggregator is not present, clear the current aggregator stats. + // For eg. if a merge is being performed, stats already collected by aggregator (numrows etc.) + // are still valid. However, if a load file is being performed, the old stats collected by + // aggregator are not valid. It might be a good idea to clear them instead of leaving wrong + // and old stats. + // Since HIVE-12661, we maintain the old stats (although may be wrong) for CBO + // purpose. We use a flag COLUMN_STATS_ACCURATE to + // show the accuracy of the stats. + + private boolean clearAggregatorStats = false; + + private boolean noStatsAggregator = false; + + private boolean isNoScanAnalyzeCommand = false; + + // sourceTask for TS is not changed (currently) but that of FS might be changed + // by various optimizers (auto.convert.join, for example) + // so this is set by DriverContext in runtime + private transient Task sourceTask; + + private boolean isFollowedByColStats = false; + + // used by FS based stats collector + private String statsTmpDir; + + public BasicStatsWork() { + } + + public BasicStatsWork(TableSpec tableSpecs) { + this.tableSpecs = tableSpecs; + } + + public BasicStatsWork(LoadTableDesc loadTableDesc) { + this.loadTableDesc = loadTableDesc; + } + + public BasicStatsWork(LoadFileDesc loadFileDesc) { + this.loadFileDesc = loadFileDesc; + } + + public TableSpec getTableSpecs() { + return tableSpecs; + } + + public LoadTableDesc getLoadTableDesc() { + return loadTableDesc; + } + + public LoadFileDesc getLoadFileDesc() { + return loadFileDesc; + } + + public void setAggKey(String aggK) { + aggKey = aggK; + } + + @Explain(displayName = "Stats Aggregation Key Prefix", explainLevels = { Level.EXTENDED }) + public String getAggKey() { + return aggKey; + } + + public String getStatsTmpDir() { + return statsTmpDir; + } + + public void setStatsTmpDir(String statsTmpDir) { + this.statsTmpDir = statsTmpDir; + } + + public boolean getNoStatsAggregator() { + return noStatsAggregator; + } + + public void setNoStatsAggregator(boolean noStatsAggregator) { + this.noStatsAggregator = noStatsAggregator; + } + + public boolean isStatsReliable() { + return statsReliable; + } + + public void setStatsReliable(boolean statsReliable) { + this.statsReliable = statsReliable; + } + + public boolean isClearAggregatorStats() { + return clearAggregatorStats; + } + + public void setClearAggregatorStats(boolean clearAggregatorStats) { + this.clearAggregatorStats = clearAggregatorStats; + } + + /** + * @return the isNoScanAnalyzeCommand + */ + public boolean isNoScanAnalyzeCommand() { + return isNoScanAnalyzeCommand; + } + + /** + * @param isNoScanAnalyzeCommand the isNoScanAnalyzeCommand to set + */ + public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) { + this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand; + } + + public Task getSourceTask() { + return sourceTask; + } + + public void setSourceTask(Task sourceTask) { + this.sourceTask = sourceTask; + } + + public boolean isFollowedByColStats1() { + return isFollowedByColStats; + } + + public void setFollowedByColStats1(boolean isFollowedByColStats) { + this.isFollowedByColStats = isFollowedByColStats; + } + + public boolean isExplicitAnalyze() { + // ANALYZE TABLE + return (getTableSpecs() != null); + } + public boolean isTargetRewritten() { + // ANALYZE TABLE + if (isExplicitAnalyze()) { + return true; + } + // INSERT OVERWRITE + if (getLoadTableDesc() != null && getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL) { + return true; + } + // CREATE TABLE ... AS + if (getLoadFileDesc() != null && getLoadFileDesc().getCtasCreateTableDesc() != null) { + return true; + } + return false; + } + + public String getTableName() { + BasicStatsWork work = this; + if (work.getLoadTableDesc() != null) { + return work.getLoadTableDesc().getTable().getTableName(); + } else if (work.getTableSpecs() != null) { + return work.getTableSpecs().tableName; + } else { + return getLoadFileDesc().getCtasCreateTableDesc().getTableName(); + } + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java index 97f323f4b7..1ca7ea6516 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java @@ -28,31 +28,25 @@ public class ColumnStatsDesc extends DDLDesc implements Serializable, Cloneable { private static final long serialVersionUID = 1L; + private FetchWork fWork; + private boolean isTblLevel; private int numBitVector; + private boolean needMerge; private String tableName; private List colName; private List colType; - public ColumnStatsDesc() { - } - public ColumnStatsDesc(String tableName, List colName, List colType, - boolean isTblLevel) { - this.tableName = tableName; - this.colName = colName; - this.colType = colType; - this.isTblLevel = isTblLevel; - this.numBitVector = 0; - } - public ColumnStatsDesc(String tableName, List colName, - List colType, boolean isTblLevel, int numBitVector) { + List colType, boolean isTblLevel, int numBitVector, FetchWork fWork1) { this.tableName = tableName; this.colName = colName; this.colType = colType; this.isTblLevel = isTblLevel; this.numBitVector = numBitVector; + this.needMerge = this.numBitVector != 0; + this.fWork = fWork1; } @Explain(displayName = "Table") @@ -99,4 +93,13 @@ public void setNumBitVector(int numBitVector) { this.numBitVector = numBitVector; } + public boolean isNeedMerge() { + return needMerge; + } + + + public FetchWork getFWork() { + return fWork; + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java deleted file mode 100644 index 842fd1a411..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.io.Serializable; - -import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.exec.ListSinkOperator; -import org.apache.hadoop.hive.ql.plan.Explain.Level; - - -/** - * ColumnStats Work. - * - */ -@Explain(displayName = "Column Stats Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class ColumnStatsWork implements Serializable { - private static final long serialVersionUID = 1L; - private FetchWork fWork; - private ColumnStatsDesc colStats; - private String currentDatabaseName; - private static final int LIMIT = -1; - - - public ColumnStatsWork() { - } - - public ColumnStatsWork(FetchWork work, ColumnStatsDesc colStats, String currentDatabaseName) { - this.fWork = work; - this.setColStats(colStats); - this.currentDatabaseName = currentDatabaseName; - } - - @Override - public String toString() { - String ret; - ret = fWork.toString(); - return ret; - } - - public FetchWork getfWork() { - return fWork; - } - - public void setfWork(FetchWork fWork) { - this.fWork = fWork; - } - - @Explain(displayName = "Column Stats Desc") - public ColumnStatsDesc getColStats() { - return colStats; - } - - public void setColStats(ColumnStatsDesc colStats) { - this.colStats = colStats; - } - - public ListSinkOperator getSink() { - return fWork.getSink(); - } - - public void initializeForFetch(CompilationOpContext ctx) { - fWork.initializeForFetch(ctx); - } - - public int getLeastNumRows() { - return fWork.getLeastNumRows(); - } - - public static int getLimit() { - return LIMIT; - } - - public String getCurrentDatabaseName() { - return currentDatabaseName; - } - - public void setCurrentDatabaseName(String currentDatabaseName) { - this.currentDatabaseName = currentDatabaseName; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index ea8fc199fa..432e37b169 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -34,7 +34,7 @@ * */ @Explain(displayName = "File Output Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class FileSinkDesc extends AbstractOperatorDesc { +public class FileSinkDesc extends AbstractOperatorDesc implements IStatsGatherDesc { private static final long serialVersionUID = 1L; public enum DPSortState { @@ -371,6 +371,7 @@ public void setGatherStats(boolean gatherStats) { this.gatherStats = gatherStats; } + @Override @Explain(displayName = "GatherStats", explainLevels = { Level.EXTENDED }) public boolean isGatherStats() { return gatherStats; @@ -387,6 +388,7 @@ public boolean isGatherStats() { * will be aggregated. * @return key prefix used for stats publishing and aggregation. */ + @Override @Explain(displayName = "Stats Publishing Key Prefix", explainLevels = { Level.EXTENDED }) public String getStatsAggPrefix() { // dirName uniquely identifies destination directory of a FileSinkOperator. @@ -509,7 +511,8 @@ public void setTable(Table table) { } - public String getStatsTmpDir() { + @Override + public String getTmpStatsDir() { return statsTmpDir; } @@ -574,4 +577,5 @@ public boolean isSame(OperatorDesc other) { } return false; } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/IStatsGatherDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/IStatsGatherDesc.java new file mode 100644 index 0000000000..a83c4fb704 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/plan/IStatsGatherDesc.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.plan; + +public interface IStatsGatherDesc { + public boolean isGatherStats(); + + String getTmpStatsDir(); + + String getStatsAggPrefix(); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java index 30d99123fc..76b9622527 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java @@ -35,7 +35,7 @@ // list of columns, comma separated private String columns; private String columnTypes; - private String destinationCreateTable; + private transient CreateTableDesc ctasCreateTableDesc; private boolean isMmCtas; public LoadFileDesc(final LoadFileDesc o) { @@ -45,24 +45,28 @@ public LoadFileDesc(final LoadFileDesc o) { this.isDfsDir = o.isDfsDir; this.columns = o.columns; this.columnTypes = o.columnTypes; - this.destinationCreateTable = o.destinationCreateTable; this.isMmCtas = o.isMmCtas; + this.ctasCreateTableDesc = o.ctasCreateTableDesc; } public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc createViewDesc, final Path sourcePath, final Path targetDir, final boolean isDfsDir, - final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) { + final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) { this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas); - if (createTableDesc != null && createTableDesc.getDatabaseName() != null - && createTableDesc.getTableName() != null) { - destinationCreateTable = (createTableDesc.getTableName().contains(".") ? "" : createTableDesc - .getDatabaseName() + ".") - + createTableDesc.getTableName(); - } else if (createViewDesc != null) { - // The work is already done in analyzeCreateView to assure that the view name is fully - // qualified. - destinationCreateTable = createViewDesc.getViewName(); + if (createTableDesc != null && createTableDesc.isCTAS()) { + ctasCreateTableDesc = createTableDesc; } + // if (createTableDesc != null && createTableDesc.getDatabaseName() != null + // && createTableDesc.getTableName() != null) { + // + // destinationCreateTable = (createTableDesc.getTableName().contains(".") ? "" : createTableDesc + // .getDatabaseName() + ".") + // + createTableDesc.getTableName(); + // } else if (createViewDesc != null) { + // // The work is already done in analyzeCreateView to assure that the view name is fully + // // qualified. + // destinationCreateTable = createViewDesc.getViewName(); + // } } public LoadFileDesc(final Path sourcePath, final Path targetDir, @@ -131,11 +135,8 @@ public void setColumnTypes(String columnTypes) { this.columnTypes = columnTypes; } - /** - * @return the destinationCreateTable - */ - public String getDestinationCreateTable(){ - return destinationCreateTable; + public CreateTableDesc getCtasCreateTableDesc() { + return ctasCreateTableDesc; } public boolean isMmCtas() { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java index 74629d51c7..b2bd4654b1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java @@ -19,136 +19,153 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.Explain.Level; - +import org.apache.hadoop.hive.ql.session.SessionState; /** - * ConditionalStats. + * Stats Work, may include basic stats work and column stats desc * */ -@Explain(displayName = "Stats-Aggr Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +@Explain(displayName = "Stats Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public class StatsWork implements Serializable { - private static final long serialVersionUID = 1L; - - private TableSpec tableSpecs; // source table spec -- for TableScanOperator - private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator - private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator - private String aggKey; // aggregation key prefix - private boolean statsReliable; // are stats completely reliable - - // If stats aggregator is not present, clear the current aggregator stats. - // For eg. if a merge is being performed, stats already collected by aggregator (numrows etc.) - // are still valid. However, if a load file is being performed, the old stats collected by - // aggregator are not valid. It might be a good idea to clear them instead of leaving wrong - // and old stats. - // Since HIVE-12661, we maintain the old stats (although may be wrong) for CBO - // purpose. We use a flag COLUMN_STATS_ACCURATE to - // show the accuracy of the stats. - private boolean clearAggregatorStats = false; - - private boolean noStatsAggregator = false; + private static final long serialVersionUID = 1L; + // this is for basic stats + private BasicStatsWork basicStatsWork; + private BasicStatsNoJobWork basicStatsNoJobWork; + private ColumnStatsDesc colStats; + private static final int LIMIT = -1; - private boolean isNoScanAnalyzeCommand = false; + private String currentDatabase; + private boolean statsReliable; + private Table table; + private boolean truncate; + private boolean footerScan; + private Set partitions = new HashSet<>(); - // sourceTask for TS is not changed (currently) but that of FS might be changed - // by various optimizers (auto.convert.join, for example) - // so this is set by DriverContext in runtime - private transient Task sourceTask; + public StatsWork(Table table, BasicStatsWork basicStatsWork, HiveConf hconf) { + super(); + this.table = table; + this.basicStatsWork = basicStatsWork; + this.currentDatabase = SessionState.get().getCurrentDatabase(); + statsReliable = hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE); + basicStatsWork.setStatsReliable(statsReliable); + } - // used by FS based stats collector - private String statsTmpDir; + public StatsWork(Table table, HiveConf hconf) { + super(); + this.table = table; + this.currentDatabase = SessionState.get().getCurrentDatabase(); + statsReliable = hconf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE); + } - public StatsWork() { + @Override + public String toString() { + return String.format("StatWork; fetch: %s", getfWork()); } - public StatsWork(TableSpec tableSpecs) { - this.tableSpecs = tableSpecs; + FetchWork getfWork() { + return colStats == null ? null : colStats.getFWork(); } - public StatsWork(LoadTableDesc loadTableDesc) { - this.loadTableDesc = loadTableDesc; + @Explain(displayName = "Column Stats Desc") + public ColumnStatsDesc getColStats() { + return colStats; } - public StatsWork(LoadFileDesc loadFileDesc) { - this.loadFileDesc = loadFileDesc; + public void setColStats(ColumnStatsDesc colStats) { + this.colStats = colStats; } - public TableSpec getTableSpecs() { - return tableSpecs; + // unused / unknown reason + @Deprecated + public static int getLimit() { + return LIMIT; } - public LoadTableDesc getLoadTableDesc() { - return loadTableDesc; + @Explain(displayName = "Basic Stats Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public BasicStatsWork getBasicStatsWork() { + return basicStatsWork; } - public LoadFileDesc getLoadFileDesc() { - return loadFileDesc; + // only explain uses it + @Explain(displayName = "Basic Stats NoJob Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public BasicStatsNoJobWork getBasicStatsNoJobWork() { + return basicStatsNoJobWork; } - public void setAggKey(String aggK) { - aggKey = aggK; + public void setSourceTask(Task sourceTask) { + basicStatsWork.setSourceTask(sourceTask); } - @Explain(displayName = "Stats Aggregation Key Prefix", explainLevels = { Level.EXTENDED }) - public String getAggKey() { - return aggKey; + public String getCurrentDatabaseName() { + return currentDatabase; } - public String getStatsTmpDir() { - return statsTmpDir; + public boolean hasColStats() { + return colStats != null; } - public void setStatsTmpDir(String statsTmpDir) { - this.statsTmpDir = statsTmpDir; + public Table getTable() { + return table; } - public boolean getNoStatsAggregator() { - return noStatsAggregator; + public void collectStatsFromAggregator(IStatsGatherDesc conf) { + // AggKey in StatsWork is used for stats aggregation while StatsAggPrefix + // in FileSinkDesc is used for stats publishing. They should be consistent. + basicStatsWork.setAggKey(conf.getStatsAggPrefix()); + basicStatsWork.setStatsTmpDir(conf.getTmpStatsDir()); + basicStatsWork.setStatsReliable(statsReliable); } - public void setNoStatsAggregator(boolean noStatsAggregator) { - this.noStatsAggregator = noStatsAggregator; + public void truncateExisting(boolean truncate) { + this.truncate = truncate; } - public boolean isStatsReliable() { - return statsReliable; + + public void setFooterScan() { + basicStatsNoJobWork = new BasicStatsNoJobWork(table.getTableSpec()); + basicStatsNoJobWork.setStatsReliable(getStatsReliable()); + footerScan = true; } - public void setStatsReliable(boolean statsReliable) { - this.statsReliable = statsReliable; + public void addInputPartitions(Set partitions) { + this.partitions.addAll(partitions); } - public boolean isClearAggregatorStats() { - return clearAggregatorStats; + public Set getPartitions() { + return partitions; } - public void setClearAggregatorStats(boolean clearAggregatorStats) { - this.clearAggregatorStats = clearAggregatorStats; + public boolean isFooterScan() { + return footerScan; } - /** - * @return the isNoScanAnalyzeCommand - */ - public boolean isNoScanAnalyzeCommand() { - return isNoScanAnalyzeCommand; + public boolean getStatsReliable() { + return statsReliable; } - /** - * @param isNoScanAnalyzeCommand the isNoScanAnalyzeCommand to set - */ - public void setNoScanAnalyzeCommand(boolean isNoScanAnalyzeCommand) { - this.isNoScanAnalyzeCommand = isNoScanAnalyzeCommand; + public String getFullTableName() { + return table.getDbName() + "." + table.getTableName(); } public Task getSourceTask() { - return sourceTask; + return basicStatsWork == null ? null : basicStatsWork.getSourceTask(); + } + + public String getAggKey() { + return basicStatsWork.getAggKey(); } - public void setSourceTask(Task sourceTask) { - this.sourceTask = sourceTask; + public boolean isAggregating() { + return basicStatsWork != null && basicStatsWork.getAggKey() != null; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index ca20afb05f..98bf817eda 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -40,7 +40,7 @@ * things will be added here as table scan is invoked as part of local work. **/ @Explain(displayName = "TableScan", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class TableScanDesc extends AbstractOperatorDesc { +public class TableScanDesc extends AbstractOperatorDesc implements IStatsGatherDesc { private static final long serialVersionUID = 1L; private String alias; @@ -260,11 +260,13 @@ public void setGatherStats(boolean gatherStats) { this.gatherStats = gatherStats; } + @Override @Explain(displayName = "GatherStats", explainLevels = { Level.EXTENDED }) public boolean isGatherStats() { return gatherStats; } + @Override public String getTmpStatsDir() { return tmpStatsDir; } @@ -293,6 +295,7 @@ public void setStatsAggPrefix(String k) { statsAggKeyPrefix = k; } + @Override @Explain(displayName = "Statistics Aggregation Key Prefix", explainLevels = { Level.EXTENDED }) public String getStatsAggPrefix() { return statsAggKeyPrefix; diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java new file mode 100644 index 0000000000..50aad2703a --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsNoJobTask.java @@ -0,0 +1,397 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import java.io.IOException; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.HiveStatsUtils; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.exec.StatsTask; +import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.plan.BasicStatsNoJobWork; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.util.StringUtils; +import org.apache.hive.common.util.ReflectionUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimaps; + +/** + * StatsNoJobTask is used in cases where stats collection is the only task for the given query (no + * parent MR or Tez job). It is used in the following cases 1) ANALYZE with noscan for + * file formats that implement StatsProvidingRecordReader interface: ORC format (implements + * StatsProvidingRecordReader) stores column statistics for all columns in the file footer. Its much + * faster to compute the table/partition statistics by reading the footer than scanning all the + * rows. This task can be used for computing basic stats like numFiles, numRows, fileSize, + * rawDataSize from ORC footer. + **/ +public class BasicStatsNoJobTask implements IStatsProcessor { + + private static transient final Logger LOG = LoggerFactory.getLogger(BasicStatsNoJobTask.class); + private HiveConf conf; + + private BasicStatsNoJobWork work; + private LogHelper console; + + public BasicStatsNoJobTask(HiveConf conf, BasicStatsNoJobWork work) { + this.conf = conf; + this.work = work; + console = new LogHelper(LOG); + } + + + @Override + public void initialize(CompilationOpContext opContext) { + + } + + @Override + public int process(Hive db, Table tbl) throws Exception { + + LOG.info("Executing stats (no job) task"); + + ExecutorService threadPool = StatsTask.newThreadPool(conf); + + return aggregateStats(threadPool, db); + } + + public StageType getType() { + return StageType.STATS; + } + + public String getName() { + return "STATS-NO-JOB"; + } + + static class StatItem { + Partish partish; + Map params; + Object result; + } + + static class FooterStatCollector implements Runnable { + + private Partish partish; + private Object result; + private JobConf jc; + private Path dir; + private FileSystem fs; + private LogHelper console; + + public FooterStatCollector(JobConf jc, Partish partish) { + this.jc = jc; + this.partish = partish; + } + + public static final Function SIMPLE_NAME_FUNCTION = new Function() { + + @Override + public String apply(FooterStatCollector sc) { + return String.format("%s#%s", sc.partish.getTable().getCompleteName(), sc.partish.getPartishType()); + } + }; + private static final Function EXTRACT_RESULT_FUNCTION = new Function() { + @Override + public Partition apply(FooterStatCollector input) { + return (Partition) input.result; + } + }; + + private boolean isValid() { + return result != null; + } + + public void init(HiveConf conf, LogHelper console) throws IOException { + this.console = console; + dir = new Path(partish.getPartSd().getLocation()); + fs = dir.getFileSystem(conf); + } + + @Override + public void run() { + + Map parameters = partish.getPartParameters(); + try { + long numRows = 0; + long rawDataSize = 0; + long fileSize = 0; + long numFiles = 0; + LOG.debug("Aggregating stats for {}", dir); + FileStatus[] fileList = HiveStatsUtils.getFileStatusRecurse(dir, -1, fs); + + for (FileStatus file : fileList) { + LOG.debug("Computing stats for {}", file); + if (!file.isDirectory()) { + InputFormat inputFormat = ReflectionUtil.newInstance(partish.getInputFormatClass(), jc); + InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { partish.getLocation() }); + if (file.getLen() == 0) { + numFiles += 1; + } else { + org.apache.hadoop.mapred.RecordReader recordReader = inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL); + try { + if (recordReader instanceof StatsProvidingRecordReader) { + StatsProvidingRecordReader statsRR; + statsRR = (StatsProvidingRecordReader) recordReader; + rawDataSize += statsRR.getStats().getRawDataSize(); + numRows += statsRR.getStats().getRowCount(); + fileSize += file.getLen(); + numFiles += 1; + } else { + throw new HiveException(String.format("Unexpected file found during reading footers for: %s ", file)); + } + } finally { + recordReader.close(); + } + } + } + } + + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); + + parameters.put(StatsSetupConst.ROW_COUNT, String.valueOf(numRows)); + parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(rawDataSize)); + parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(fileSize)); + parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(numFiles)); + + if (partish.getPartition() != null) { + result = new Partition(partish.getTable(), partish.getPartition().getTPartition()); + } else { + result = new Table(partish.getTable().getTTable()); + } + + // printout console and debug logs + String threadName = Thread.currentThread().getName(); + // partish.getSimpleName(); + String msg = partish.getSimpleName() + " stats: [" + toString(parameters) + ']'; + LOG.debug(threadName + ": " + msg); + console.printInfo(msg); + + } catch (Exception e) { + console.printInfo("[Warning] could not update stats for " + partish.getSimpleName() + ".", "Failed with exception " + e.getMessage() + "\n" + StringUtils.stringifyException(e)); + if (isStatsReliable()) { + // XXX: possibly a bad idea? + // notify somehow! + // throw new HiveException("Could not update stats for: " + partish.getSimpleName(), e); + } + } + } + + private boolean isStatsReliable() { + return false; + } + + private String toString(Map parameters) { + StringBuilder builder = new StringBuilder(); + for (String statType : StatsSetupConst.supportedStats) { + String value = parameters.get(statType); + if (value != null) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(statType).append('=').append(value); + } + } + return builder.toString(); + } + + } + + private int aggregateStats(ExecutorService threadPool, Hive db) { + int ret = 0; + try { + JobConf jc = new JobConf(conf); + + TableSpec tableSpecs = work.getTableSpecs(); + + if (tableSpecs == null) { + throw new RuntimeException("this is unexpected...needs some investigation"); + } + + Table table = tableSpecs.tableHandle; + + Collection partitions = null; + if (work.getPartitions() == null || work.getPartitions().isEmpty()) { + if (table.isPartitioned()) { + partitions = tableSpecs.partitions; + } + } else { + partitions = work.getPartitions(); + } + + LinkedList partishes = Lists.newLinkedList(); + if (partitions == null) { + partishes.add(Partish.buildFor(table)); + } else { + for (Partition part : partitions) { + partishes.add(Partish.buildFor(table, part)); + } + } + + List scs = Lists.newArrayList(); + for (Partish partish : partishes) { + scs.add(new FooterStatCollector(jc, partish)); + } + + for (FooterStatCollector sc : scs) { + sc.init(conf, console); + threadPool.execute(sc); + } + + LOG.debug("Stats collection waiting for threadpool to shutdown.."); + shutdownAndAwaitTermination(threadPool); + LOG.debug("Stats collection threadpool shutdown successful."); + + ret = updatePartitions(db, scs, table); + + } catch (Exception e) { + console.printError("Failed to collect footer statistics.", "Failed with exception " + e.getMessage() + "\n" + StringUtils.stringifyException(e)); + // Fail the query if the stats are supposed to be reliable + if (work.isStatsReliable()) { + ret = -1; + } + } + + // The return value of 0 indicates success, + // anything else indicates failure + return ret; + } + + private int updatePartitions(Hive db, List scs, Table table) throws InvalidOperationException, HiveException { + + String tableFullName = table.getFullyQualifiedName(); + + if (scs.isEmpty()) { + return 0; + } + if (work.isStatsReliable()) { + for (FooterStatCollector statsCollection : scs) { + if (statsCollection.result == null) { + LOG.debug("Stats requested to be reliable. Empty stats found: {}", statsCollection.partish.getSimpleName()); + return -1; + } + } + } + List validColectors = Lists.newArrayList(); + for (FooterStatCollector statsCollection : scs) { + if (statsCollection.isValid()) { + validColectors.add(statsCollection); + } + } + + EnvironmentContext environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + + ImmutableListMultimap collectorsByTable = Multimaps.index(validColectors, FooterStatCollector.SIMPLE_NAME_FUNCTION); + + LOG.debug("Collectors.size(): {}", collectorsByTable.keySet()); + + if (collectorsByTable.keySet().size() < 1) { + LOG.warn("Collectors are empty! ; {}", tableFullName); + } + + // for now this should be true... + assert (collectorsByTable.keySet().size() <= 1); + + LOG.debug("Updating stats for: {}", tableFullName); + + for (String partName : collectorsByTable.keySet()) { + ImmutableList values = collectorsByTable.get(partName); + + if (values == null) { + throw new RuntimeException("very intresting"); + } + + if (values.get(0).result instanceof Table) { + db.alterTable(tableFullName, (Table) values.get(0).result, environmentContext); + LOG.debug("Updated stats for {}.", tableFullName); + } else { + if (values.get(0).result instanceof Partition) { + List results = Lists.transform(values, FooterStatCollector.EXTRACT_RESULT_FUNCTION); + db.alterPartitions(tableFullName, results, environmentContext); + LOG.debug("Bulk updated {} partitions of {}.", results.size(), tableFullName); + } else { + throw new RuntimeException("inconsistent"); + } + } + } + LOG.debug("Updated stats for: {}", tableFullName); + return 0; + } + + private void shutdownAndAwaitTermination(ExecutorService threadPool) { + + // Disable new tasks from being submitted + threadPool.shutdown(); + try { + + // Wait a while for existing tasks to terminate + // XXX this will wait forever... :) + while (!threadPool.awaitTermination(10, TimeUnit.SECONDS)) { + LOG.debug("Waiting for all stats tasks to finish..."); + } + // Cancel currently executing tasks + threadPool.shutdownNow(); + + // Wait a while for tasks to respond to being cancelled + if (!threadPool.awaitTermination(100, TimeUnit.SECONDS)) { + LOG.debug("Stats collection thread pool did not terminate"); + } + } catch (InterruptedException ie) { + + // Cancel again if current thread also interrupted + threadPool.shutdownNow(); + + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + } + + @Override + public void setDpPartSpecs(Collection dpPartSpecs) { + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java new file mode 100644 index 0000000000..f995f387a8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -0,0 +1,509 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hive.ql.stats; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.plan.BasicStatsWork; +import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; +import org.apache.hadoop.hive.ql.plan.LoadTableDesc; +import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * StatsTask implementation. StatsTask mainly deals with "collectable" stats. These are + * stats that require data scanning and are collected during query execution (unless the user + * explicitly requests data scanning just for the purpose of stats computation using the "ANALYZE" + * command. All other stats are computed directly by the MetaStore. The rationale being that the + * MetaStore layer covers all Thrift calls and provides better guarantees about the accuracy of + * those stats. + **/ +public class BasicStatsTask implements Serializable, IStatsProcessor { + + private static final long serialVersionUID = 1L; + private static transient final Logger LOG = LoggerFactory.getLogger(BasicStatsTask.class); + + private Table table; + private Collection dpPartSpecs; + public boolean followedColStats; + private BasicStatsWork work; + private HiveConf conf; + + protected transient LogHelper console; + + public BasicStatsTask(HiveConf conf, BasicStatsWork work) { + super(); + dpPartSpecs = null; + this.conf = conf; + console = new LogHelper(LOG); + this.work = work; + } + + @Override + public int process(Hive db, Table tbl) throws Exception { + + LOG.info("Executing stats task"); + table = tbl; + return aggregateStats(db); + } + + @Override + public void initialize(CompilationOpContext opContext) { + } + + public StageType getType() { + return StageType.STATS; + } + + public String getName() { + return "STATS"; + } + + private static class BasicStatsProcessor { + + private Partish partish; + private FileStatus[] partfileStatus; + private BasicStatsWork work; + private boolean atomic; + private boolean followedColStats1; + + public BasicStatsProcessor(Partish partish, BasicStatsWork work, HiveConf conf, boolean followedColStats2) { + this.partish = partish; + this.work = work; + atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC); + followedColStats1 = followedColStats2; + } + + public Object process(StatsAggregator statsAggregator) throws HiveException, MetaException { + Partish p = partish; + Map parameters = p.getPartParameters(); + if (p.isAcid()) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); + } + + if (work.isTargetRewritten()) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); + } + + // work.getTableSpecs() == null means it is not analyze command + // and then if it is not followed by column stats, we should clean + // column stats + // FIXME: move this to ColStat related part + if (!work.isExplicitAnalyze() && !followedColStats1) { + StatsSetupConst.clearColumnStatsState(parameters); + } + // non-partitioned tables: + // XXX: I don't aggree with this logic + // FIXME: deprecate atomic? what's its purpose? + if (!existStats(parameters) && atomic) { + return null; + } + if(partfileStatus == null){ + LOG.warn("Partition/partfiles is null for: " + partish.getPartition().getSpec()); + return null; + } + + // The collectable stats for the aggregator needs to be cleared. + // For eg. if a file is being loaded, the old number of rows are not valid + // XXX: makes no sense for me... possibly not needed anymore + if (work.isClearAggregatorStats()) { + // we choose to keep the invalid stats and only change the setting. + // FIXME: invalidating numRows would be preferred here...instead keeping anything + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); + removeStatsRelatedParams(parameters, StatsSetupConst.supportedStats); + } + + updateQuickStats(parameters, partfileStatus); + if (StatsSetupConst.areBasicStatsUptoDate(parameters)) { + if (statsAggregator != null) { + String prefix = getAggregationPrefix(p.getTable(), p.getPartition()); + updateStats(statsAggregator, parameters, prefix, atomic); + } + } else { + removeStatsRelatedParams(parameters, StatsSetupConst.statsRequireCompute); + } + + return p.getOutput(); + } + + private void removeStatsRelatedParams(Map parameters, String[] statsrequirecompute) { + for (String string : statsrequirecompute) { + parameters.remove(string); + } + + } + public void collectFileStatus(Warehouse wh) throws MetaException { + Map parameters = partish.getPartParameters(); + if (!existStats(parameters) && atomic) { + return; + } + partfileStatus = wh.getFileStatusesForSD(partish.getPartSd()); + } + + @Deprecated + private boolean existStats(Map parameters) { + return parameters.containsKey(StatsSetupConst.ROW_COUNT) + || parameters.containsKey(StatsSetupConst.NUM_FILES) + || parameters.containsKey(StatsSetupConst.TOTAL_SIZE) + || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE) + || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS); + } + + private void updateQuickStats(Map parameters, FileStatus[] partfileStatus) throws MetaException { + MetaStoreUtils.populateQuickStats(partfileStatus, parameters); + } + + private String getAggregationPrefix(Table table, Partition partition) throws MetaException { + String prefix = getAggregationPrefix0(table, partition); + String aggKey = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR; + return aggKey; + } + + private String getAggregationPrefix0(Table table, Partition partition) throws MetaException { + + // prefix is of the form dbName.tblName + String prefix = table.getDbName() + "." + org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(table.getTableName()); + // FIXME: this is a secret contract; reusein getAggrKey() creates a more closer relation to the StatsGatherer + // prefix = work.getAggKey(); + prefix = prefix.toLowerCase(); + if (partition != null) { + return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); + } + return prefix; + } + + private void updateStats(StatsAggregator statsAggregator, Map parameters, String aggKey, boolean atomic) throws HiveException { + + for (String statType : StatsSetupConst.statsRequireCompute) { + String value = statsAggregator.aggregateStats(aggKey, statType); + if (value != null && !value.isEmpty()) { + long longValue = Long.parseLong(value); + + if (!work.isTargetRewritten()) { + String originalValue = parameters.get(statType); + if (originalValue != null) { + longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid + } + } + parameters.put(statType, String.valueOf(longValue)); + } else { + if (atomic) { + throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType); + } + } + } + } + + } + + + private int aggregateStats(Hive db) { + + StatsAggregator statsAggregator = null; + int ret = 0; + StatsCollectionContext scc = null; + EnvironmentContext environmentContext = null; + environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + + try { + // Stats setup: + final Warehouse wh = new Warehouse(conf); + if (!getWork().getNoStatsAggregator() && !getWork().isNoScanAnalyzeCommand()) { + try { + scc = getContext(); + statsAggregator = createStatsAggregator(scc, conf); + } catch (HiveException e) { + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_RELIABLE)) { + throw e; + } + console.printError(ErrorMsg.STATS_SKIPPING_BY_ERROR.getErrorCodedMsg(e.toString())); + } + } + + List partitions = getPartitionsList(db); + + String tableFullName = table.getDbName() + "." + table.getTableName(); + + List partishes = new ArrayList<>(); + + if (partitions == null) { + Partish p; + partishes.add(p = new Partish.PTable(table)); + + BasicStatsProcessor basicStatsProcessor = new BasicStatsProcessor(p, work, conf, followedColStats); + basicStatsProcessor.collectFileStatus(wh); + Object res = basicStatsProcessor.process(statsAggregator); + + if (res == null) { + return 0; + } + db.alterTable(tableFullName, (Table) res, environmentContext); + + if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { + console.printInfo("Table " + tableFullName + " stats: [" + toString(p.getPartParameters()) + ']'); + } + LOG.info("Table " + tableFullName + " stats: [" + toString(p.getPartParameters()) + ']'); + + } else { + // Partitioned table: + // Need to get the old stats of the partition + // and update the table stats based on the old and new stats. + + List updates = new ArrayList(); + + final ExecutorService pool = buildBasicStatsExecutor(); + + final List> futures = Lists.newLinkedList(); + List processors = Lists.newLinkedList(); + + try { + for(final Partition partn : partitions) { + Partish p; + BasicStatsProcessor bsp = new BasicStatsProcessor(p = new Partish.PPart(table, partn), work, conf, followedColStats); + processors.add(bsp); + + futures.add(pool.submit(new Callable() { + @Override + public Void call() throws Exception { + bsp.collectFileStatus(wh); + return null; + } + })); + } + pool.shutdown(); + for (Future future : futures) { + future.get(); + } + } catch (InterruptedException e) { + LOG.debug("Cancelling " + futures.size() + " file stats lookup tasks"); + //cancel other futures + for (Future future : futures) { + future.cancel(true); + } + // Fail the query if the stats are supposed to be reliable + if (work.isStatsReliable()) { + ret = 1; + } + } finally { + if (pool != null) { + pool.shutdownNow(); + } + LOG.debug("Finished getting file stats of all partitions!"); + } + + for (BasicStatsProcessor basicStatsProcessor : processors) { + Object res = basicStatsProcessor.process(statsAggregator); + if (res == null) { + LOG.info("Partition " + basicStatsProcessor.partish.getPartition().getSpec() + " stats: [0]"); + continue; + } + updates.add((Partition) res); + if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) { + console.printInfo("Partition " + basicStatsProcessor.partish.getPartition().getSpec() + " stats: [" + toString(basicStatsProcessor.partish.getPartParameters()) + ']'); + } + LOG.info("Partition " + basicStatsProcessor.partish.getPartition().getSpec() + " stats: [" + toString(basicStatsProcessor.partish.getPartParameters()) + ']'); + } + + if (!updates.isEmpty()) { + db.alterPartitions(tableFullName, updates, environmentContext); + } + if (work.isStatsReliable() && updates.size() != processors.size()) { + LOG.info("Stats should be reliadble...however seems like there were some issue.. => ret 1"); + ret = 1; + } + } + + } catch (Exception e) { + console.printInfo("[Warning] could not update stats.", + "Failed with exception " + e.getMessage() + "\n" + + StringUtils.stringifyException(e)); + + // Fail the query if the stats are supposed to be reliable + if (work.isStatsReliable()) { + ret = 1; + } + } finally { + if (statsAggregator != null) { + statsAggregator.closeConnection(scc); + } + } + // The return value of 0 indicates success, + // anything else indicates failure + return ret; + } + + private BasicStatsWork getWork() { + return work; + } + + private ExecutorService buildBasicStatsExecutor() { + //Get the file status up-front for all partitions. Beneficial in cases of blob storage systems + int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 1); + // In case thread count is set to 0, use single thread. + poolSize = Math.max(poolSize, 1); + final ExecutorService pool = Executors.newFixedThreadPool(poolSize, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("stats-updater-thread-%d").build()); + LOG.debug("Getting file stats of all partitions. threadpool size:" + poolSize); + return pool; + } + + private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { + String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); + StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); + if (factory == null) { + throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); + } + // initialize stats publishing table for noscan which has only stats task + // the rest of MR task following stats task initializes it in ExecDriver.java + StatsPublisher statsPublisher = factory.getStatsPublisher(); + if (!statsPublisher.init(scc)) { // creating stats table if not exists + throw new HiveException(ErrorMsg.STATSPUBLISHER_INITIALIZATION_ERROR.getErrorCodedMsg()); + } + + // manufacture a StatsAggregator + StatsAggregator statsAggregator = factory.getStatsAggregator(); + if (!statsAggregator.connect(scc)) { + throw new HiveException(ErrorMsg.STATSAGGREGATOR_CONNECTION_ERROR.getErrorCodedMsg(statsImpl)); + } + return statsAggregator; + } + + private StatsCollectionContext getContext() throws HiveException { + + StatsCollectionContext scc = new StatsCollectionContext(conf); + Task sourceTask = getWork().getSourceTask(); + if (sourceTask == null) { + throw new HiveException(ErrorMsg.STATSAGGREGATOR_SOURCETASK_NULL.getErrorCodedMsg()); + } + scc.setTask(sourceTask); + scc.setStatsTmpDir(this.getWork().getStatsTmpDir()); + return scc; + } + + + private String toString(Map parameters) { + StringBuilder builder = new StringBuilder(); + for (String statType : StatsSetupConst.supportedStats) { + String value = parameters.get(statType); + if (value != null) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(statType).append('=').append(value); + } + } + return builder.toString(); + } + + /** + * Get the list of partitions that need to update statistics. + * TODO: we should reuse the Partitions generated at compile time + * since getting the list of partitions is quite expensive. + * + * @return a list of partitions that need to update statistics. + * @throws HiveException + */ + private List getPartitionsList(Hive db) throws HiveException { + if (work.getLoadFileDesc() != null) { + return null; //we are in CTAS, so we know there are no partitions + } + + List list = new ArrayList(); + + if (work.getTableSpecs() != null) { + + // ANALYZE command + TableSpec tblSpec = work.getTableSpecs(); + table = tblSpec.tableHandle; + if (!table.isPartitioned()) { + return null; + } + // get all partitions that matches with the partition spec + List partitions = tblSpec.partitions; + if (partitions != null) { + for (Partition partn : partitions) { + list.add(partn); + } + } + } else if (work.getLoadTableDesc() != null) { + + // INSERT OVERWRITE command + LoadTableDesc tbd = work.getLoadTableDesc(); + table = db.getTable(tbd.getTable().getTableName()); + if (!table.isPartitioned()) { + return null; + } + DynamicPartitionCtx dpCtx = tbd.getDPCtx(); + if (dpCtx != null && dpCtx.getNumDPCols() > 0) { // dynamic partitions + // If no dynamic partitions are generated, dpPartSpecs may not be initialized + if (dpPartSpecs != null) { + // load the list of DP partitions and return the list of partition specs + list.addAll(dpPartSpecs); + } + } else { // static partition + Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); + list.add(partn); + } + } + return list; + } + + public Collection getDpPartSpecs() { + return dpPartSpecs; + } + + @Override + public void setDpPartSpecs(Collection dpPartSpecs) { + this.dpPartSpecs = dpPartSpecs; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java new file mode 100644 index 0000000000..7ce7a746a0 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.exec.FetchOperator; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; +import org.apache.hadoop.hive.ql.plan.FetchWork; +import org.apache.hadoop.hive.ql.stats.ColumnStatisticsObjTranslator; +import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ColStatsProcessor implements IStatsProcessor { + private static transient final Logger LOG = LoggerFactory.getLogger(ColStatsProcessor.class); + + private FetchOperator ftOp; + private FetchWork fWork; + private ColumnStatsDesc colStatDesc; + private HiveConf conf; + private boolean isStatsReliable; + + public ColStatsProcessor(ColumnStatsDesc colStats, HiveConf conf) { + this.conf = conf; + fWork = colStats.getFWork(); + colStatDesc = colStats; + isStatsReliable = conf.getBoolVar(ConfVars.HIVE_STATS_RELIABLE); + } + + @Override + public void initialize(CompilationOpContext opContext) { + try { + fWork.initializeForFetch(opContext); + JobConf job = new JobConf(conf); + ftOp = new FetchOperator(fWork, job); + } catch (Exception e) { + LOG.error(StringUtils.stringifyException(e)); + throw new RuntimeException(e); + } + } + + @Override + public int process(Hive db, Table tbl) throws Exception { + return persistColumnStats(db, tbl); + } + + private List constructColumnStatsFromPackedRows(Table tbl1) throws HiveException, MetaException, IOException { + + Table tbl = tbl1; + + String partName = null; + List colName = colStatDesc.getColName(); + List colType = colStatDesc.getColType(); + boolean isTblLevel = colStatDesc.isTblLevel(); + + List stats = new ArrayList(); + InspectableObject packedRow; + while ((packedRow = ftOp.getNextRow()) != null) { + if (packedRow.oi.getCategory() != ObjectInspector.Category.STRUCT) { + throw new HiveException("Unexpected object type encountered while unpacking row"); + } + + List statsObjs = new ArrayList(); + StructObjectInspector soi = (StructObjectInspector) packedRow.oi; + List fields = soi.getAllStructFieldRefs(); + List list = soi.getStructFieldsDataAsList(packedRow.o); + + List partColSchema = tbl.getPartCols(); + // Partition columns are appended at end, we only care about stats column + int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size(); + assert list != null; + for (int i = 0; i < numOfStatCols; i++) { + StructField structField = fields.get(i); + String columnName = colName.get(i); + String columnType = colType.get(i); + Object values = list.get(i); + try { + ColumnStatisticsObj statObj = ColumnStatisticsObjTranslator.readHiveStruct(columnName, columnType, structField, values); + statsObjs.add(statObj); + } catch (Exception e) { + if (isStatsReliable) { + throw new HiveException("Statistics collection failed while (hive.stats.reliable)", e); + } else { + LOG.debug("Because {} is infinite or NaN, we skip stats.", columnName, e); + } + } + } + + if (!statsObjs.isEmpty()) { + + if (!isTblLevel) { + List partVals = new ArrayList(); + // Iterate over partition columns to figure out partition name + for (int i = fields.size() - partColSchema.size(); i < fields.size(); i++) { + Object partVal = ((PrimitiveObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(list.get(i)); + partVals.add(partVal == null ? // could be null for default partition + this.conf.getVar(ConfVars.DEFAULTPARTITIONNAME) : partVal.toString()); + } + partName = Warehouse.makePartName(partColSchema, partVals); + } + + ColumnStatisticsDesc statsDesc = buildColumnStatsDesc(tbl, partName, isTblLevel); + ColumnStatistics colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + stats.add(colStats); + } + } + ftOp.clearFetchContext(); + return stats; + } + + private ColumnStatisticsDesc buildColumnStatsDesc(Table table, String partName, boolean isTblLevel) { + String dbName = table.getDbName(); + assert dbName != null; + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(table.getTableName()); + statsDesc.setIsTblLevel(isTblLevel); + + if (!isTblLevel) { + statsDesc.setPartName(partName); + } else { + statsDesc.setPartName(null); + } + return statsDesc; + } + + public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaException, IOException { + // Construct a column statistics object from the result + + List colStats = constructColumnStatsFromPackedRows(tbl); + // Persist the column statistics object to the metastore + // Note, this function is shared for both table and partition column stats. + if (colStats.isEmpty()) { + return 0; + } + SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); + request.setNeedMerge(colStatDesc.isNeedMerge()); + db.setPartitionColumnStatistics(request); + return 0; + } + + @Override + public void setDpPartSpecs(Collection dpPartSpecs) { + } + +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java similarity index 59% rename from ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java rename to ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java index 1f286887e4..648552625f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java @@ -16,45 +16,25 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.exec; +package org.apache.hadoop.hive.ql.stats; -import java.io.IOException; -import java.io.Serializable; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Date; import org.apache.hadoop.hive.metastore.api.Decimal; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; -import org.apache.hadoop.hive.ql.CompilationOpContext; -import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.QueryPlan; -import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; -import org.apache.hadoop.hive.ql.plan.ColumnStatsWork; -import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -65,40 +45,28 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** - * ColumnStatsTask implementation. - **/ - -public class ColumnStatsTask extends Task implements Serializable { - private static final long serialVersionUID = 1L; - private FetchOperator ftOp; - private static transient final Logger LOG = LoggerFactory.getLogger(ColumnStatsTask.class); +public class ColumnStatisticsObjTranslator { - public ColumnStatsTask() { - super(); - } - - @Override - public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx, - CompilationOpContext opContext) { - super.initialize(queryState, queryPlan, ctx, opContext); - work.initializeForFetch(opContext); + public static ColumnStatisticsObj readHiveStruct(String columnName, String columnType, StructField structField, Object values) + throws HiveException + { + // Get the field objectInspector, fieldName and the field object. + ObjectInspector foi = structField.getFieldObjectInspector(); + Object f = values; + String fieldName = structField.getFieldName(); + ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); + statsObj.setColName(columnName); + statsObj.setColType(columnType); try { - JobConf job = new JobConf(conf); - ftOp = new FetchOperator(work.getfWork(), job); + unpackStructObject(foi, f, fieldName, statsObj); + return statsObj; } catch (Exception e) { - LOG.error(StringUtils.stringifyException(e)); - throw new RuntimeException(e); + throw new HiveException("error calculating stats for column:" + structField.getFieldName(), e); } } - private void unpackBooleanStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { + private static void unpackBooleanStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { long v = ((LongObjectInspector) oi).get(o); if (fName.equals("counttrues")) { statsObj.getStatsData().getBooleanStats().setNumTrues(v); @@ -110,11 +78,10 @@ private void unpackBooleanStats(ObjectInspector oi, Object o, String fName, } @SuppressWarnings("serial") - class UnsupportedDoubleException extends Exception { + static class UnsupportedDoubleException extends Exception { } - private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { + private static void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); @@ -137,11 +104,11 @@ private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(buf); + ; } } - private void unpackDecimalStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { + private static void unpackDecimalStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDecimalStats().setNumNulls(v); @@ -158,15 +125,15 @@ private void unpackDecimalStats(ObjectInspector oi, Object o, String fName, PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setBitVectors(buf); + ; } } - private Decimal convertToThriftDecimal(HiveDecimal d) { - return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), (short)d.scale()); + private static Decimal convertToThriftDecimal(HiveDecimal d) { + return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), (short) d.scale()); } - private void unpackLongStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { + private static void unpackLongStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumNulls(v); @@ -177,17 +144,17 @@ private void unpackLongStats(ObjectInspector oi, Object o, String fName, long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setHighValue(v); } else if (fName.equals("min")) { - long v = ((LongObjectInspector) oi).get(o); + long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setLowValue(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getLongStats().setBitVectors(buf); + ; } } - private void unpackStringStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { + private static void unpackStringStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumNulls(v); @@ -204,11 +171,11 @@ private void unpackStringStats(ObjectInspector oi, Object o, String fName, PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getStringStats().setBitVectors(buf); + ; } } - private void unpackBinaryStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { + private static void unpackBinaryStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setNumNulls(v); @@ -221,8 +188,7 @@ private void unpackBinaryStats(ObjectInspector oi, Object o, String fName, } } - private void unpackDateStats(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj statsObj) { + private static void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); @@ -239,11 +205,11 @@ private void unpackDateStats(ObjectInspector oi, Object o, String fName, PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(buf); + ; } } - private void unpackPrimitiveObject (ObjectInspector oi, Object o, String fieldName, - ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { + private static void unpackPrimitiveObject(ObjectInspector oi, Object o, String fieldName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (o == null) { return; } @@ -289,7 +255,7 @@ private void unpackPrimitiveObject (ObjectInspector oi, Object o, String fieldNa } else if (statsObj.getStatsData().isSetLongStats()) { unpackLongStats(oi, o, fieldName, statsObj); } else if (statsObj.getStatsData().isSetDoubleStats()) { - unpackDoubleStats(oi,o,fieldName, statsObj); + unpackDoubleStats(oi, o, fieldName, statsObj); } else if (statsObj.getStatsData().isSetStringStats()) { unpackStringStats(oi, o, fieldName, statsObj); } else if (statsObj.getStatsData().isSetBinaryStats()) { @@ -302,8 +268,7 @@ private void unpackPrimitiveObject (ObjectInspector oi, Object o, String fieldNa } } - private void unpackStructObject(ObjectInspector oi, Object o, String fName, - ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { + private static void unpackStructObject(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { if (oi.getCategory() != ObjectInspector.Category.STRUCT) { throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString()); } @@ -325,127 +290,4 @@ private void unpackStructObject(ObjectInspector oi, Object o, String fName, } } } - - private List constructColumnStatsFromPackedRows( - Hive db) throws HiveException, MetaException, IOException { - - String currentDb = work.getCurrentDatabaseName(); - String tableName = work.getColStats().getTableName(); - String partName = null; - List colName = work.getColStats().getColName(); - List colType = work.getColStats().getColType(); - boolean isTblLevel = work.getColStats().isTblLevel(); - - List stats = new ArrayList(); - InspectableObject packedRow; - Table tbl = db.getTable(currentDb, tableName); - while ((packedRow = ftOp.getNextRow()) != null) { - if (packedRow.oi.getCategory() != ObjectInspector.Category.STRUCT) { - throw new HiveException("Unexpected object type encountered while unpacking row"); - } - - List statsObjs = new ArrayList(); - StructObjectInspector soi = (StructObjectInspector) packedRow.oi; - List fields = soi.getAllStructFieldRefs(); - List list = soi.getStructFieldsDataAsList(packedRow.o); - - List partColSchema = tbl.getPartCols(); - // Partition columns are appended at end, we only care about stats column - int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size(); - for (int i = 0; i < numOfStatCols; i++) { - // Get the field objectInspector, fieldName and the field object. - ObjectInspector foi = fields.get(i).getFieldObjectInspector(); - Object f = (list == null ? null : list.get(i)); - String fieldName = fields.get(i).getFieldName(); - ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); - statsObj.setColName(colName.get(i)); - statsObj.setColType(colType.get(i)); - try { - unpackStructObject(foi, f, fieldName, statsObj); - statsObjs.add(statsObj); - } catch (UnsupportedDoubleException e) { - // due to infinity or nan. - LOG.info("Because {} is infinite or NaN, we skip stats.", colName.get(i)); - } - } - - if (!isTblLevel) { - List partVals = new ArrayList(); - // Iterate over partition columns to figure out partition name - for (int i = fields.size() - partColSchema.size(); i < fields.size(); i++) { - Object partVal = ((PrimitiveObjectInspector)fields.get(i).getFieldObjectInspector()). - getPrimitiveJavaObject(list.get(i)); - partVals.add(partVal == null ? // could be null for default partition - this.conf.getVar(ConfVars.DEFAULTPARTITIONNAME) : partVal.toString()); - } - partName = Warehouse.makePartName(partColSchema, partVals); - } - String [] names = Utilities.getDbTableName(currentDb, tableName); - ColumnStatisticsDesc statsDesc = getColumnStatsDesc(names[0], names[1], partName, isTblLevel); - ColumnStatistics colStats = new ColumnStatistics(); - colStats.setStatsDesc(statsDesc); - colStats.setStatsObj(statsObjs); - if (!statsObjs.isEmpty()) { - stats.add(colStats); - } - } - ftOp.clearFetchContext(); - return stats; - } - - private ColumnStatisticsDesc getColumnStatsDesc(String dbName, String tableName, - String partName, boolean isTblLevel) - { - ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); - statsDesc.setDbName(dbName); - statsDesc.setTableName(tableName); - statsDesc.setIsTblLevel(isTblLevel); - - if (!isTblLevel) { - statsDesc.setPartName(partName); - } else { - statsDesc.setPartName(null); - } - return statsDesc; - } - - private int persistColumnStats(Hive db) throws HiveException, MetaException, IOException { - // Construct a column statistics object from the result - List colStats = constructColumnStatsFromPackedRows(db); - // Persist the column statistics object to the metastore - // Note, this function is shared for both table and partition column stats. - if (colStats.isEmpty()) { - return 0; - } - SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); - if (work.getColStats() != null && work.getColStats().getNumBitVector() > 0) { - request.setNeedMerge(true); - } - db.setPartitionColumnStatistics(request); - return 0; - } - - @Override - public int execute(DriverContext driverContext) { - if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { - return 0; - } - try { - Hive db = getHive(); - return persistColumnStats(db); - } catch (Exception e) { - LOG.error("Failed to run column stats task", e); - } - return 1; - } - - @Override - public StageType getType() { - return StageType.COLUMNSTATS; - } - - @Override - public String getName() { - return "COLUMNSTATS TASK"; - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/IStatsProcessor.java ql/src/java/org/apache/hadoop/hive/ql/stats/IStatsProcessor.java new file mode 100644 index 0000000000..04219b5e21 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/IStatsProcessor.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import java.util.Collection; + +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +public interface IStatsProcessor { + + void initialize(CompilationOpContext opContext); + + int process(Hive db, Table tbl) throws Exception; + + void setDpPartSpecs(Collection dpPartSpecs); + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/stats/Partish.java ql/src/java/org/apache/hadoop/hive/ql/stats/Partish.java new file mode 100644 index 0000000000..e8d3184f40 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/stats/Partish.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.stats; + +import java.util.Map; + +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.OutputFormat; + +/** + * Cover class to make it easier to make modifications on partitions/tables + */ +public abstract class Partish { + + public static Partish buildFor(Table table) { + return new PTable(table); + } + + public static Partish buildFor(Partition part) { + return new PPart(part.getTable(), part); + } + + public static Partish buildFor(Table table, Partition part) { + return new PPart(table, part); + } + + // rename + @Deprecated + public final boolean isAcid() { + return AcidUtils.isFullAcidTable(getTable()); + } + + public abstract Table getTable(); + + public abstract Map getPartParameters(); + + public abstract StorageDescriptor getPartSd(); + + public abstract Object getOutput() throws HiveException; + + public abstract Partition getPartition(); + + public abstract Class getInputFormatClass() throws HiveException; + + public abstract Class getOutputFormatClass() throws HiveException; + + public abstract String getLocation(); + + public abstract String getSimpleName(); + + public final String getPartishType() { + return getClass().getSimpleName(); + } + + static class PTable extends Partish { + private Table table; + + public PTable(Table table) { + this.table = table; + } + + @Override + public Table getTable() { + return table; + } + + @Override + public Map getPartParameters() { + return table.getTTable().getParameters(); + } + + @Override + public StorageDescriptor getPartSd() { + return table.getTTable().getSd(); + } + + @Override + public Object getOutput() throws HiveException { + return new Table(getTable().getTTable()); + } + + @Override + public Partition getPartition() { + return null; + } + + @Override + public Class getInputFormatClass() { + return table.getInputFormatClass(); + } + + @Override + public Class getOutputFormatClass() { + return table.getOutputFormatClass(); + } + + @Override + public String getLocation() { + return table.getDataLocation().toString(); + } + + @Override + public String getSimpleName() { + return String.format("Table %s.%s", table.getDbName(), table.getTableName()); + } + } + + static class PPart extends Partish { + private Table table; + private Partition partition; + + // FIXME: possibly the distinction between table/partition is not need; however it was like this before....will change it later + public PPart(Table table, Partition partiton) { + this.table = table; + partition = partiton; + } + + @Override + public Table getTable() { + return table; + } + + @Override + public Map getPartParameters() { + return partition.getTPartition().getParameters(); + } + + @Override + public StorageDescriptor getPartSd() { + return partition.getTPartition().getSd(); + } + + @Override + public Object getOutput() throws HiveException { + return new Partition(table, partition.getTPartition()); + } + + @Override + public Partition getPartition() { + return partition; + } + + @Override + public Class getInputFormatClass() throws HiveException { + return partition.getInputFormatClass(); + } + + @Override + public Class getOutputFormatClass() throws HiveException { + return partition.getOutputFormatClass(); + } + + @Override + public String getLocation() { + return partition.getLocation(); + } + + @Override + public String getSimpleName() { + return String.format("Partition %s.%s %s", table.getDbName(), table.getTableName(), partition.getSpec()); + } + + } + +} \ No newline at end of file diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 39d6b2b414..832b10bdc2 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -833,4 +833,4 @@ public void testMoreBucketsThanReducers2() throws Exception { int[][] expected = {{0, -1},{0, -1}, {1, -1}, {1, -1}, {2, -1}, {2, -1}, {3, -1}, {3, -1}}; Assert.assertEquals(stringifyValues(expected), r); } -} \ No newline at end of file +} diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 3737b6a4d5..9342eb35ff 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -133,6 +133,7 @@ protected void setUpWithTableProperties(String tableProperties) throws Exception .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); TxnDbUtil.setConfValues(hiveConf); TxnDbUtil.prepDb(hiveConf); diff --git ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index 3e4f6f6675..8737369c39 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -80,6 +80,7 @@ void setUpInternal() throws Exception { .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); TxnDbUtil.setConfValues(hiveConf); TxnDbUtil.prepDb(hiveConf); File f = new File(getWarehouseDir()); diff --git ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveReduceExpressionsWithStatsRule.java ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveReduceExpressionsWithStatsRule.java index d0a9982244..4c865e0d08 100644 --- ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveReduceExpressionsWithStatsRule.java +++ ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/rules/TestHiveReduceExpressionsWithStatsRule.java @@ -92,7 +92,7 @@ public void before() { builder = HiveRelFactories.HIVE_BUILDER.create(optCluster, schemaMock); - StatsSetupConst.setStatsStateForCreateTable(tableParams, Lists.newArrayList("_int"), "TRUE"); + StatsSetupConst.setStatsStateForCreateTable(tableParams, Lists.newArrayList("_int"), StatsSetupConst.TRUE); tableParams.put(StatsSetupConst.ROW_COUNT, "3"); } diff --git ql/src/test/queries/clientpositive/autoColumnStats_1.q ql/src/test/queries/clientpositive/autoColumnStats_1.q index 7955b07233..cc32393e65 100644 --- ql/src/test/queries/clientpositive/autoColumnStats_1.q +++ ql/src/test/queries/clientpositive/autoColumnStats_1.q @@ -60,6 +60,8 @@ drop table nzhang_part14; create table if not exists nzhang_part14 (key string) partitioned by (value string); +desc formatted nzhang_part14; + insert overwrite table nzhang_part14 partition(value) select key, value from ( select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a @@ -69,6 +71,8 @@ select key, value from ( select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T; +desc formatted nzhang_part14 partition (value=' '); + explain select key from nzhang_part14; diff --git ql/src/test/queries/clientpositive/autoColumnStats_10.q ql/src/test/queries/clientpositive/autoColumnStats_10.q new file mode 100644 index 0000000000..bf166d8701 --- /dev/null +++ ql/src/test/queries/clientpositive/autoColumnStats_10.q @@ -0,0 +1,52 @@ +set hive.mapred.mode=nonstrict; +set hive.stats.column.autogather=true; + +drop table p; + +CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint); + +desc formatted p; + +insert into p values (1,22,333); + +desc formatted p; + +alter table p replace columns (insert_num int, c1 STRING, c2 STRING); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; + +insert into p values (2,11,111); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; + +set hive.stats.column.autogather=false; + +drop table p; + +CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint); + +desc formatted p; + +insert into p values (1,22,333); + +desc formatted p; + +alter table p replace columns (insert_num int, c1 STRING, c2 STRING); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; + +insert into p values (2,11,111); + +desc formatted p; + +desc formatted p insert_num; +desc formatted p c1; diff --git ql/src/test/queries/clientpositive/autoColumnStats_5a.q ql/src/test/queries/clientpositive/autoColumnStats_5a.q new file mode 100644 index 0000000000..a8bce18e92 --- /dev/null +++ ql/src/test/queries/clientpositive/autoColumnStats_5a.q @@ -0,0 +1,30 @@ +set hive.stats.column.autogather=true; +set hive.mapred.mode=nonstrict; +set hive.cli.print.header=true; +SET hive.exec.schema.evolution=true; +SET hive.vectorized.execution.enabled=false; +set hive.fetch.task.conversion=none; +set hive.exec.dynamic.partition.mode=nonstrict; + +-- SORT_QUERY_RESULTS + +CREATE TABLE partitioned1(a INT, b STRING) PARTITIONED BY(part INT) STORED AS TEXTFILE; + +explain extended +insert into table partitioned1 partition(part=1) values(1, 'original'); + +insert into table partitioned1 partition(part=1) values(1, 'original'); + +desc formatted partitioned1 partition(part=1); + +explain extended +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original'); + +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original'); + +explain insert into table partitioned1 partition(part=1) values(1, 'original'),(2, 'original'), (3, 'original'),(4, 'original'); + +desc formatted partitioned1; +desc formatted partitioned1 partition(part=1); +desc formatted partitioned1 partition(part=1) a; + diff --git ql/src/test/queries/clientpositive/basicstat_partval.q ql/src/test/queries/clientpositive/basicstat_partval.q new file mode 100644 index 0000000000..2db472d3f3 --- /dev/null +++ ql/src/test/queries/clientpositive/basicstat_partval.q @@ -0,0 +1,12 @@ +set hive.stats.autogather=true; + +CREATE TABLE p1(i int) partitioned by (p string); + +insert into p1 partition(p='a') values (1); +insert into p1 partition(p='A') values (2),(3); + +describe formatted p1; +describe formatted p1 partition(p='a'); +describe formatted p1 partition(p='A'); + + diff --git ql/src/test/queries/clientpositive/columnstats_partlvl.q ql/src/test/queries/clientpositive/columnstats_partlvl.q index 2c92dfef4f..4283bca1f5 100644 --- ql/src/test/queries/clientpositive/columnstats_partlvl.q +++ ql/src/test/queries/clientpositive/columnstats_partlvl.q @@ -14,6 +14,8 @@ explain extended analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns employeeID; analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns employeeID; +describe formatted Employee_Part partition(employeeSalary=2000.0); + explain analyze table Employee_Part partition (employeeSalary=4000.0) compute statistics for columns employeeID; explain extended diff --git ql/src/test/queries/clientpositive/columnstats_partlvl_dp.q ql/src/test/queries/clientpositive/columnstats_partlvl_dp.q index ead9a2d7b8..c065edd3dc 100644 --- ql/src/test/queries/clientpositive/columnstats_partlvl_dp.q +++ ql/src/test/queries/clientpositive/columnstats_partlvl_dp.q @@ -16,6 +16,8 @@ explain analyze table Employee_Part partition (employeeSalary='4000.0', country) compute statistics for columns employeeName, employeeID; analyze table Employee_Part partition (employeeSalary='4000.0', country) compute statistics for columns employeeName, employeeID; +describe formatted Employee_Part partition (employeeSalary='4000.0', country='USA'); + describe formatted Employee_Part partition (employeeSalary='4000.0', country='USA') employeeName; -- don't specify all partitioning keys diff --git ql/src/test/queries/clientpositive/deleteAnalyze.q ql/src/test/queries/clientpositive/deleteAnalyze.q index 26123a64bb..5293ddfb36 100644 --- ql/src/test/queries/clientpositive/deleteAnalyze.q +++ ql/src/test/queries/clientpositive/deleteAnalyze.q @@ -20,6 +20,8 @@ describe formatted testdeci2 amount; analyze table testdeci2 compute statistics for columns; +describe formatted testdeci2; + set hive.stats.fetch.column.stats=true; analyze table testdeci2 compute statistics for columns; diff --git ql/src/test/queries/clientpositive/exec_parallel_column_stats.q ql/src/test/queries/clientpositive/exec_parallel_column_stats.q index ceacc2442d..a89b707918 100644 --- ql/src/test/queries/clientpositive/exec_parallel_column_stats.q +++ ql/src/test/queries/clientpositive/exec_parallel_column_stats.q @@ -1,5 +1,7 @@ set hive.exec.parallel=true; -explain analyze table src compute statistics for columns; +create table t as select * from src; -analyze table src compute statistics for columns; \ No newline at end of file +explain analyze table t compute statistics for columns; + +analyze table t compute statistics for columns; diff --git ql/src/test/queries/clientpositive/outer_reference_windowed.q ql/src/test/queries/clientpositive/outer_reference_windowed.q index cac6b413d9..3259ebe8c5 100644 --- ql/src/test/queries/clientpositive/outer_reference_windowed.q +++ ql/src/test/queries/clientpositive/outer_reference_windowed.q @@ -34,6 +34,8 @@ ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS; set hive.explain.user=false; +describe formatted e011_01; + explain select sum(sum(c1)) over() from e011_01; select sum(sum(c1)) over() from e011_01; diff --git ql/src/test/queries/clientpositive/smb_mapjoin_1.q ql/src/test/queries/clientpositive/smb_mapjoin_1.q index baf1690106..b2394ad10f 100644 --- ql/src/test/queries/clientpositive/smb_mapjoin_1.q +++ ql/src/test/queries/clientpositive/smb_mapjoin_1.q @@ -12,6 +12,9 @@ load data local inpath '../../data/files/smbbucket_1.rc' overwrite into table sm load data local inpath '../../data/files/smbbucket_2.rc' overwrite into table smb_bucket_2; load data local inpath '../../data/files/smbbucket_3.rc' overwrite into table smb_bucket_3; +desc formatted smb_bucket_1; +select count(*) from smb_bucket_1; + set hive.cbo.enable=false; set hive.optimize.bucketmapjoin = true; set hive.optimize.bucketmapjoin.sortedmerge = true; diff --git ql/src/test/queries/clientpositive/temp_table_display_colstats_tbllvl.q ql/src/test/queries/clientpositive/temp_table_display_colstats_tbllvl.q index 39a11f27eb..e1215831f3 100644 --- ql/src/test/queries/clientpositive/temp_table_display_colstats_tbllvl.q +++ ql/src/test/queries/clientpositive/temp_table_display_colstats_tbllvl.q @@ -15,8 +15,11 @@ CREATE TEMPORARY EXTERNAL TABLE UserVisits_web_text_none ( row format delimited fields terminated by '|' stored as textfile location 'pfile://${system:test.tmp.dir}/uservisits_web_text_none'; +desc formatted UserVisits_web_text_none; + LOAD DATA LOCAL INPATH "../../data/files/UserVisits.dat" INTO TABLE UserVisits_web_text_none; +desc formatted UserVisits_web_text_none; desc extended UserVisits_web_text_none sourceIP; desc formatted UserVisits_web_text_none sourceIP; @@ -27,6 +30,7 @@ explain extended analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue; analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue; +desc formatted UserVisits_web_text_none; desc formatted UserVisits_web_text_none sourceIP; desc formatted UserVisits_web_text_none avgTimeOnSite; desc formatted UserVisits_web_text_none adRevenue; diff --git ql/src/test/results/clientpositive/acid_table_stats.q.out ql/src/test/results/clientpositive/acid_table_stats.q.out index 32c8531754..8fad7710a3 100644 --- ql/src/test/results/clientpositive/acid_table_stats.q.out +++ ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -91,9 +91,8 @@ Database: default Table: acid #### A masked pattern was here #### Partition Parameters: + COLUMN_STATS_ACCURATE {} numFiles 2 - numRows 0 - rawDataSize 0 totalSize 3950 #### A masked pattern was here #### @@ -226,11 +225,15 @@ PREHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@acid PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid +PREHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@acid POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid +POSTHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: desc formatted acid partition(ds='2008-04-08') PREHOOK: type: DESCTABLE @@ -378,9 +381,8 @@ Database: default Table: acid #### A masked pattern was here #### Partition Parameters: + COLUMN_STATS_ACCURATE {} numFiles 4 - numRows 1000 - rawDataSize 208000 totalSize 7904 #### A masked pattern was here #### @@ -502,11 +504,15 @@ PREHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@acid PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid +PREHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table acid partition(ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@acid POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid +POSTHOOK: Output: default@acid@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: explain select max(key) from acid where ds='2008-04-08' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/alterColumnStatsPart.q.out ql/src/test/results/clientpositive/alterColumnStatsPart.q.out index 6b9b7e5407..858e16fe22 100644 --- ql/src/test/results/clientpositive/alterColumnStatsPart.q.out +++ ql/src/test/results/clientpositive/alterColumnStatsPart.q.out @@ -154,11 +154,15 @@ PREHOOK: query: analyze table p partition(c=1) compute statistics for columns a PREHOOK: type: QUERY PREHOOK: Input: default@p PREHOOK: Input: default@p@c=1 +PREHOOK: Output: default@p +PREHOOK: Output: default@p@c=1 #### A masked pattern was here #### POSTHOOK: query: analyze table p partition(c=1) compute statistics for columns a POSTHOOK: type: QUERY POSTHOOK: Input: default@p POSTHOOK: Input: default@p@c=1 +POSTHOOK: Output: default@p +POSTHOOK: Output: default@p@c=1 #### A masked pattern was here #### PREHOOK: query: explain select max(a) from p where c=1 PREHOOK: type: QUERY @@ -371,11 +375,15 @@ PREHOOK: query: analyze table p partition(c=100) compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@p PREHOOK: Input: default@p@c=100 +PREHOOK: Output: default@p +PREHOOK: Output: default@p@c=100 #### A masked pattern was here #### POSTHOOK: query: analyze table p partition(c=100) compute statistics for columns a POSTHOOK: type: QUERY POSTHOOK: Input: default@p POSTHOOK: Input: default@p@c=100 +POSTHOOK: Output: default@p +POSTHOOK: Output: default@p@c=100 #### A masked pattern was here #### PREHOOK: query: explain select max(a) from p where c=100 PREHOOK: type: QUERY @@ -474,11 +482,15 @@ PREHOOK: query: analyze table p partition(c=100) compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@p PREHOOK: Input: default@p@c=100 +PREHOOK: Output: default@p +PREHOOK: Output: default@p@c=100 #### A masked pattern was here #### POSTHOOK: query: analyze table p partition(c=100) compute statistics for columns a POSTHOOK: type: QUERY POSTHOOK: Input: default@p POSTHOOK: Input: default@p@c=100 +POSTHOOK: Output: default@p +POSTHOOK: Output: default@p@c=100 #### A masked pattern was here #### PREHOOK: query: explain select max(a) from p where c=100 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/alter_partition_update_status.q.out ql/src/test/results/clientpositive/alter_partition_update_status.q.out index 37b24edf49..9aab9b2436 100644 --- ql/src/test/results/clientpositive/alter_partition_update_status.q.out +++ ql/src/test/results/clientpositive/alter_partition_update_status.q.out @@ -22,11 +22,15 @@ PREHOOK: query: ANALYZE TABLE src_stat_part_one PARTITION(partitionId=1) COMPUTE PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part_one PREHOOK: Input: default@src_stat_part_one@partitionid=1 +PREHOOK: Output: default@src_stat_part_one +PREHOOK: Output: default@src_stat_part_one@partitionid=1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part_one PARTITION(partitionId=1) COMPUTE STATISTICS for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part_one POSTHOOK: Input: default@src_stat_part_one@partitionid=1 +POSTHOOK: Output: default@src_stat_part_one +POSTHOOK: Output: default@src_stat_part_one@partitionid=1 #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part_one PARTITION(partitionId=1) key PREHOOK: type: DESCTABLE @@ -92,11 +96,15 @@ PREHOOK: query: ANALYZE TABLE src_stat_part_two PARTITION(px=1) COMPUTE STATISTI PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part_two PREHOOK: Input: default@src_stat_part_two@px=1/py=a +PREHOOK: Output: default@src_stat_part_two +PREHOOK: Output: default@src_stat_part_two@px=1/py=a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part_two PARTITION(px=1) COMPUTE STATISTICS for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part_two POSTHOOK: Input: default@src_stat_part_two@px=1/py=a +POSTHOOK: Output: default@src_stat_part_two +POSTHOOK: Output: default@src_stat_part_two@px=1/py=a #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part_two PARTITION(px=1, py='a') key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/alter_table_column_stats.q.out ql/src/test/results/clientpositive/alter_table_column_stats.q.out index a648335195..a8ef59c0b8 100644 --- ql/src/test/results/clientpositive/alter_table_column_stats.q.out +++ ql/src/test/results/clientpositive/alter_table_column_stats.q.out @@ -77,10 +77,12 @@ PREHOOK: query: analyze table testtable0 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable0 POSTHOOK: query: analyze table testtable0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable0 PREHOOK: query: describe formatted statsdb1.testtable0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable0 @@ -585,12 +587,18 @@ PREHOOK: Input: statsdb1@testpart0 PREHOOK: Input: statsdb1@testpart0@part=part1 PREHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testpart0 +PREHOOK: Output: statsdb1@testpart0@part=part1 +PREHOOK: Output: statsdb1@testpart0@part=part2 POSTHOOK: query: analyze table testpart0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testpart0 POSTHOOK: Input: statsdb1@testpart0@part=part1 POSTHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testpart0 +POSTHOOK: Output: statsdb1@testpart0@part=part1 +POSTHOOK: Output: statsdb1@testpart0@part=part2 PREHOOK: query: describe formatted statsdb1.testpart0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testpart0 @@ -1939,10 +1947,12 @@ PREHOOK: query: analyze table testtable0 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable0 POSTHOOK: query: analyze table testtable0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable0 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable0 PREHOOK: query: describe formatted statsdb1.testtable0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable0 @@ -2447,12 +2457,18 @@ PREHOOK: Input: statsdb1@testpart0 PREHOOK: Input: statsdb1@testpart0@part=part1 PREHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testpart0 +PREHOOK: Output: statsdb1@testpart0@part=part1 +PREHOOK: Output: statsdb1@testpart0@part=part2 POSTHOOK: query: analyze table testpart0 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testpart0 POSTHOOK: Input: statsdb1@testpart0@part=part1 POSTHOOK: Input: statsdb1@testpart0@part=part2 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testpart0 +POSTHOOK: Output: statsdb1@testpart0@part=part1 +POSTHOOK: Output: statsdb1@testpart0@part=part2 PREHOOK: query: describe formatted statsdb1.testpart0 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testpart0 diff --git ql/src/test/results/clientpositive/alter_table_update_status.q.out ql/src/test/results/clientpositive/alter_table_update_status.q.out index 899ebbd34d..c6140be263 100644 --- ql/src/test/results/clientpositive/alter_table_update_status.q.out +++ ql/src/test/results/clientpositive/alter_table_update_status.q.out @@ -35,10 +35,12 @@ POSTHOOK: Output: default@src_stat_int PREHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat +PREHOOK: Output: default@src_stat #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat +POSTHOOK: Output: default@src_stat #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat key PREHOOK: type: DESCTABLE @@ -108,10 +110,12 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true PREHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_int +PREHOOK: Output: default@src_stat_int #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_int +POSTHOOK: Output: default@src_stat_int #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_int key PREHOOK: type: DESCTABLE @@ -131,7 +135,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') PREHOOK: type: ALTERTABLE_UPDATETABLESTATS POSTHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') @@ -154,7 +158,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: create database if not exists dummydb PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:dummydb diff --git ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out index 8e4ca0aca3..5ff136d6d2 100644 --- ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out +++ ql/src/test/results/clientpositive/alter_table_update_status_disable_bitvector.q.out @@ -35,10 +35,12 @@ POSTHOOK: Output: default@src_stat_int PREHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat +PREHOOK: Output: default@src_stat #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat +POSTHOOK: Output: default@src_stat #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat key PREHOOK: type: DESCTABLE @@ -108,10 +110,12 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true PREHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_int +PREHOOK: Output: default@src_stat_int #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_int COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_int +POSTHOOK: Output: default@src_stat_int #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_int key PREHOOK: type: DESCTABLE @@ -131,7 +135,7 @@ num_trues num_falses bitVector comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') PREHOOK: type: ALTERTABLE_UPDATETABLESTATS POSTHOOK: query: ALTER TABLE src_stat_int UPDATE STATISTICS for column key SET ('numDVs'='2222','lowValue'='333.22','highValue'='22.22') @@ -154,7 +158,7 @@ num_trues num_falses bitVector comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"key\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} PREHOOK: query: create database if not exists dummydb PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:dummydb diff --git ql/src/test/results/clientpositive/analyze_tbl_date.q.out ql/src/test/results/clientpositive/analyze_tbl_date.q.out index 6b2b85be23..e70c6b8bf0 100644 --- ql/src/test/results/clientpositive/analyze_tbl_date.q.out +++ ql/src/test/results/clientpositive/analyze_tbl_date.q.out @@ -16,10 +16,12 @@ POSTHOOK: Lineage: test_table.d EXPRESSION [(values__tmp__table__1)values__tmp__ PREHOOK: query: analyze table test_table compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@test_table +PREHOOK: Output: default@test_table #### A masked pattern was here #### POSTHOOK: query: analyze table test_table compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table +POSTHOOK: Output: default@test_table #### A masked pattern was here #### PREHOOK: query: describe formatted test_table PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/analyze_tbl_part.q.out ql/src/test/results/clientpositive/analyze_tbl_part.q.out index 0f38045a01..5899ab2af0 100644 --- ql/src/test/results/clientpositive/analyze_tbl_part.q.out +++ ql/src/test/results/clientpositive/analyze_tbl_part.q.out @@ -35,12 +35,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part PREHOOK: Input: default@src_stat_part@partitionid=1 PREHOOK: Input: default@src_stat_part@partitionid=2 +PREHOOK: Output: default@src_stat_part +PREHOOK: Output: default@src_stat_part@partitionid=1 +PREHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part partition (partitionId) COMPUTE STATISTICS for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part POSTHOOK: Input: default@src_stat_part@partitionid=1 POSTHOOK: Input: default@src_stat_part@partitionid=2 +POSTHOOK: Output: default@src_stat_part +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) key PREHOOK: type: DESCTABLE @@ -65,12 +71,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_part PREHOOK: Input: default@src_stat_part@partitionid=1 PREHOOK: Input: default@src_stat_part@partitionid=2 +PREHOOK: Output: default@src_stat_part +PREHOOK: Output: default@src_stat_part@partitionid=1 +PREHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_part partition (partitionId) COMPUTE STATISTICS for columns key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_part POSTHOOK: Input: default@src_stat_part@partitionid=1 POSTHOOK: Input: default@src_stat_part@partitionid=2 +POSTHOOK: Output: default@src_stat_part +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 #### A masked pattern was here #### PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) key PREHOOK: type: DESCTABLE @@ -144,19 +156,27 @@ PREHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\' PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_string_part PREHOOK: Input: default@src_stat_string_part@partitionname=p%271 +PREHOOK: Output: default@src_stat_string_part +PREHOOK: Output: default@src_stat_string_part@partitionname=p%271 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\'1") COMPUTE STATISTICS for columns key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_string_part POSTHOOK: Input: default@src_stat_string_part@partitionname=p%271 +POSTHOOK: Output: default@src_stat_string_part +POSTHOOK: Output: default@src_stat_string_part@partitionname=p%271 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\"1") COMPUTE STATISTICS for columns key, value PREHOOK: type: QUERY PREHOOK: Input: default@src_stat_string_part PREHOOK: Input: default@src_stat_string_part@partitionname=p%221 +PREHOOK: Output: default@src_stat_string_part +PREHOOK: Output: default@src_stat_string_part@partitionname=p%221 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src_stat_string_part partition (partitionName="p\"1") COMPUTE STATISTICS for columns key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stat_string_part POSTHOOK: Input: default@src_stat_string_part@partitionname=p%221 +POSTHOOK: Output: default@src_stat_string_part +POSTHOOK: Output: default@src_stat_string_part@partitionname=p%221 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out index da59211f81..ab8c22b9be 100644 --- ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out +++ ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out @@ -59,10 +59,12 @@ POSTHOOK: Output: default@over1k PREHOOK: query: analyze table over1k compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@over1k +PREHOOK: Output: default@over1k #### A masked pattern was here #### POSTHOOK: query: analyze table over1k compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@over1k +POSTHOOK: Output: default@over1k #### A masked pattern was here #### PREHOOK: query: explain select count(*) from over1k where ( (t=1 and si=2) diff --git ql/src/test/results/clientpositive/annotate_stats_filter.q.out ql/src/test/results/clientpositive/annotate_stats_filter.q.out index defb0880f3..20f870ea94 100644 --- ql/src/test/results/clientpositive/annotate_stats_filter.q.out +++ ql/src/test/results/clientpositive/annotate_stats_filter.q.out @@ -112,10 +112,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select * from loc_orc where state='OH' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_groupby.q.out ql/src/test/results/clientpositive/annotate_stats_groupby.q.out index cd4b0ad3a6..4e67841080 100644 --- ql/src/test/results/clientpositive/annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/annotate_stats_groupby.q.out @@ -76,10 +76,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select a, c, min(b) from ( select state as a, locid as b, count(*) as c @@ -177,10 +179,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select year from loc_orc group by year PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out index 49868793ca..ffcb20fc06 100644 --- ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out +++ ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out @@ -29,10 +29,12 @@ POSTHOOK: Output: default@location PREHOOK: query: analyze table location compute statistics for columns state, country PREHOOK: type: QUERY PREHOOK: Input: default@location +PREHOOK: Output: default@location #### A masked pattern was here #### POSTHOOK: query: analyze table location compute statistics for columns state, country POSTHOOK: type: QUERY POSTHOOK: Input: default@location +POSTHOOK: Output: default@location #### A masked pattern was here #### PREHOOK: query: explain select state, country from location group by state, country PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_join.q.out ql/src/test/results/clientpositive/annotate_stats_join.q.out index a7e73a06c7..9173e7acf1 100644 --- ql/src/test/results/clientpositive/annotate_stats_join.q.out +++ ql/src/test/results/clientpositive/annotate_stats_join.q.out @@ -97,26 +97,32 @@ POSTHOOK: Output: default@loc PREHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid PREHOOK: type: QUERY PREHOOK: Input: default@emp +PREHOOK: Output: default@emp #### A masked pattern was here #### POSTHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp +POSTHOOK: Output: default@emp #### A masked pattern was here #### PREHOOK: query: analyze table dept compute statistics for columns deptname,deptid PREHOOK: type: QUERY PREHOOK: Input: default@dept +PREHOOK: Output: default@dept #### A masked pattern was here #### POSTHOOK: query: analyze table dept compute statistics for columns deptname,deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@dept +POSTHOOK: Output: default@dept #### A masked pattern was here #### PREHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc +PREHOOK: Output: default@loc #### A masked pattern was here #### POSTHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc +POSTHOOK: Output: default@loc #### A masked pattern was here #### PREHOOK: query: explain select * from emp e join dept d on (e.deptid = d.deptid) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out index a73e34d724..bec98d0874 100644 --- ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out +++ ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out @@ -297,10 +297,12 @@ POSTHOOK: Output: default@store PREHOOK: query: analyze table store compute statistics for columns s_store_sk, s_floor_space PREHOOK: type: QUERY PREHOOK: Input: default@store +PREHOOK: Output: default@store #### A masked pattern was here #### POSTHOOK: query: analyze table store compute statistics for columns s_store_sk, s_floor_space POSTHOOK: type: QUERY POSTHOOK: Input: default@store +POSTHOOK: Output: default@store #### A masked pattern was here #### PREHOOK: query: analyze table store_bigint compute statistics PREHOOK: type: QUERY @@ -313,10 +315,12 @@ POSTHOOK: Output: default@store_bigint PREHOOK: query: analyze table store_bigint compute statistics for columns s_store_sk, s_floor_space PREHOOK: type: QUERY PREHOOK: Input: default@store_bigint +PREHOOK: Output: default@store_bigint #### A masked pattern was here #### POSTHOOK: query: analyze table store_bigint compute statistics for columns s_store_sk, s_floor_space POSTHOOK: type: QUERY POSTHOOK: Input: default@store_bigint +POSTHOOK: Output: default@store_bigint #### A masked pattern was here #### PREHOOK: query: analyze table store_sales compute statistics PREHOOK: type: QUERY @@ -329,10 +333,12 @@ POSTHOOK: Output: default@store_sales PREHOOK: query: analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity PREHOOK: type: QUERY PREHOOK: Input: default@store_sales +PREHOOK: Output: default@store_sales #### A masked pattern was here #### POSTHOOK: query: analyze table store_sales compute statistics for columns ss_store_sk, ss_addr_sk, ss_quantity POSTHOOK: type: QUERY POSTHOOK: Input: default@store_sales +POSTHOOK: Output: default@store_sales #### A masked pattern was here #### PREHOOK: query: analyze table customer_address compute statistics PREHOOK: type: QUERY @@ -345,10 +351,12 @@ POSTHOOK: Output: default@customer_address PREHOOK: query: analyze table customer_address compute statistics for columns ca_address_sk PREHOOK: type: QUERY PREHOOK: Input: default@customer_address +PREHOOK: Output: default@customer_address #### A masked pattern was here #### POSTHOOK: query: analyze table customer_address compute statistics for columns ca_address_sk POSTHOOK: type: QUERY POSTHOOK: Input: default@customer_address +POSTHOOK: Output: default@customer_address #### A masked pattern was here #### PREHOOK: query: explain select s.s_store_sk from store s join store_sales ss on (s.s_store_sk = ss.ss_store_sk) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_limit.q.out ql/src/test/results/clientpositive/annotate_stats_limit.q.out index ea181cb8f6..5139db4d64 100644 --- ql/src/test/results/clientpositive/annotate_stats_limit.q.out +++ ql/src/test/results/clientpositive/annotate_stats_limit.q.out @@ -55,10 +55,12 @@ POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name PREHOOK: query: analyze table loc_orc compute statistics for columns state, locid, zip, year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state, locid, zip, year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select * from loc_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_part.q.out ql/src/test/results/clientpositive/annotate_stats_part.q.out index def4d4faee..3a94a6a4e3 100644 --- ql/src/test/results/clientpositive/annotate_stats_part.q.out +++ ql/src/test/results/clientpositive/annotate_stats_part.q.out @@ -273,11 +273,15 @@ PREHOOK: query: analyze table loc_orc partition(year='2001') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc PREHOOK: Input: default@loc_orc@year=2001 +PREHOOK: Output: default@loc_orc +PREHOOK: Output: default@loc_orc@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc partition(year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc POSTHOOK: Input: default@loc_orc@year=2001 +POSTHOOK: Output: default@loc_orc +POSTHOOK: Output: default@loc_orc@year=2001 #### A masked pattern was here #### PREHOOK: query: explain select zip from loc_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_select.q.out ql/src/test/results/clientpositive/annotate_stats_select.q.out index e10ce1a12b..9aaa6f68a2 100644 --- ql/src/test/results/clientpositive/annotate_stats_select.q.out +++ ql/src/test/results/clientpositive/annotate_stats_select.q.out @@ -113,10 +113,12 @@ STAGE PLANS: PREHOOK: query: analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1 PREHOOK: type: QUERY PREHOOK: Input: default@alltypes_orc +PREHOOK: Output: default@alltypes_orc #### A masked pattern was here #### POSTHOOK: query: analyze table alltypes_orc compute statistics for columns bo1, ti1, si1, i1, bi1, f1, d1, s1, vc1 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_orc +POSTHOOK: Output: default@alltypes_orc #### A masked pattern was here #### PREHOOK: query: explain select * from alltypes_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_table.q.out ql/src/test/results/clientpositive/annotate_stats_table.q.out index ff7b403da3..f61e8d831a 100644 --- ql/src/test/results/clientpositive/annotate_stats_table.q.out +++ ql/src/test/results/clientpositive/annotate_stats_table.q.out @@ -120,10 +120,12 @@ STAGE PLANS: PREHOOK: query: analyze table emp_orc compute statistics for columns deptid PREHOOK: type: QUERY PREHOOK: Input: default@emp_orc +PREHOOK: Output: default@emp_orc #### A masked pattern was here #### POSTHOOK: query: analyze table emp_orc compute statistics for columns deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_orc +POSTHOOK: Output: default@emp_orc #### A masked pattern was here #### PREHOOK: query: explain select * from emp_orc PREHOOK: type: QUERY @@ -170,10 +172,12 @@ STAGE PLANS: PREHOOK: query: analyze table emp_orc compute statistics for columns lastname,deptid PREHOOK: type: QUERY PREHOOK: Input: default@emp_orc +PREHOOK: Output: default@emp_orc #### A masked pattern was here #### POSTHOOK: query: analyze table emp_orc compute statistics for columns lastname,deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp_orc +POSTHOOK: Output: default@emp_orc #### A masked pattern was here #### PREHOOK: query: explain select * from emp_orc PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/annotate_stats_union.q.out ql/src/test/results/clientpositive/annotate_stats_union.q.out index 059f261d97..3b4b169dc8 100644 --- ql/src/test/results/clientpositive/annotate_stats_union.q.out +++ ql/src/test/results/clientpositive/annotate_stats_union.q.out @@ -55,10 +55,12 @@ POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select state from loc_orc PREHOOK: type: QUERY @@ -282,18 +284,22 @@ PREHOOK: query: analyze table loc_staging compute statistics for columns state,l PREHOOK: type: QUERY PREHOOK: Input: test@loc_staging #### A masked pattern was here #### +PREHOOK: Output: test@loc_staging POSTHOOK: query: analyze table loc_staging compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: test@loc_staging #### A masked pattern was here #### +POSTHOOK: Output: test@loc_staging PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: test@loc_orc #### A masked pattern was here #### +PREHOOK: Output: test@loc_orc POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: test@loc_orc #### A masked pattern was here #### +POSTHOOK: Output: test@loc_orc PREHOOK: query: explain select * from (select state from default.loc_orc union all select state from test.loc_orc) temp PREHOOK: type: QUERY POSTHOOK: query: explain select * from (select state from default.loc_orc union all select state from test.loc_orc) temp diff --git ql/src/test/results/clientpositive/autoColumnStats_1.q.out ql/src/test/results/clientpositive/autoColumnStats_1.q.out new file mode 100644 index 0000000000..e04eaf4530 --- /dev/null +++ ql/src/test/results/clientpositive/autoColumnStats_1.q.out @@ -0,0 +1,1433 @@ +PREHOOK: query: drop table src_multi1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_multi1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_multi1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: insert overwrite table src_multi1 select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: insert overwrite table src_multi1 select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Lineage: src_multi1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended select * from src_multi1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select * from src_multi1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_multi1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: describe formatted src_multi1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi1 +POSTHOOK: query: describe formatted src_multi1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi1 +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table a like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: from src +insert overwrite table a select * +insert overwrite table b select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +PREHOOK: Output: default@b +POSTHOOK: query: from src +insert overwrite table a select * +insert overwrite table b select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Output: default@b +POSTHOOK: Lineage: a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@a +POSTHOOK: query: describe formatted a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@a +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted b +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@a +PREHOOK: Output: default@a +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@a +POSTHOOK: Output: default@a +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@b +PREHOOK: Output: default@b +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@b +POSTHOOK: Output: default@b +PREHOOK: query: create table a like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: from src +insert overwrite table a select * +insert into table b select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +PREHOOK: Output: default@b +POSTHOOK: query: from src +insert overwrite table a select * +insert into table b select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Output: default@b +POSTHOOK: Lineage: a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@a +POSTHOOK: query: describe formatted a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@a +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted b +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table src_multi2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_multi2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_multi2 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_multi2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi2 +POSTHOOK: query: describe formatted src_multi2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi2 +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 508 + rawDataSize 5400 + totalSize 5908 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table if not exists nzhang_part14 (key string) + partitioned by (value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string) + partitioned by (value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: desc formatted nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string + +# Partition Information +# col_name data_type comment +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from ( + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a + union all + select * from (select 'k2' as key, '' as value from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value from src limit 2)c +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from ( + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a + union all + select * from (select 'k2' as key, '' as value from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value from src limit 2)c +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_part14@value= +POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: nzhang_part14 PARTITION(value= ).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] +PREHOOK: query: desc formatted nzhang_part14 partition (value=' ') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 partition (value=' ') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string + +# Partition Information +# col_name data_type comment +value string + +# Detailed Partition Information +Partition Value: [ ] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 4 + totalSize 6 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select key from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 6 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table src5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src5 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src5 as select key, value from src limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src5 +POSTHOOK: query: create table src5 as select key, value from src limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5 +POSTHOOK: Lineage: src5.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src5.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from src5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src5 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from src5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src5 +POSTHOOK: Output: default@nzhang_part14@value=val_165 +POSTHOOK: Output: default@nzhang_part14@value=val_238 +POSTHOOK: Output: default@nzhang_part14@value=val_27 +POSTHOOK: Output: default@nzhang_part14@value=val_311 +POSTHOOK: Output: default@nzhang_part14@value=val_86 +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_165).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_238).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_27).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_311).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_86).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: explain select key from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL + ListSink + +PREHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alter5 +POSTHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter5 +PREHOOK: query: alter table alter5 add partition (dt='a') location 'parta' +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@alter5 +POSTHOOK: query: alter table alter5 add partition (dt='a') location 'parta' +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@alter5 +POSTHOOK: Output: default@alter5@dt=a +PREHOOK: query: describe formatted alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe formatted alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +# col_name data_type comment +col1 string + +# Partition Information +# col_name data_type comment +dt string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: alter5 +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert overwrite table alter5 partition (dt='a') select key from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@alter5@dt=a +POSTHOOK: query: insert overwrite table alter5 partition (dt='a') select key from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@alter5@dt=a +POSTHOOK: Lineage: alter5 PARTITION(dt=a).col1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: describe formatted alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe formatted alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +# col_name data_type comment +col1 string + +# Partition Information +# col_name data_type comment +dt string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: alter5 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 1406 + totalSize 1906 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select * from alter5 where dt='a' +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from alter5 where dt='a' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: alter5 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col1 (type: string), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table src_stat_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_stat_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_stat_part(key string, value string) partitioned by (partitionId int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_stat_part +POSTHOOK: query: create table src_stat_part(key string, value string) partitioned by (partitionId int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_stat_part +PREHOOK: query: insert overwrite table src_stat_part partition (partitionId=1) +select * from src1 limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: query: insert overwrite table src_stat_part partition (partitionId=1) +select * from src1 limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=1).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=1).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_stat_part +POSTHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_stat_part +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +partitionid int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: src_stat_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 5 + rawDataSize 38 + totalSize 43 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert overwrite table src_stat_part partition (partitionId=2) +select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_stat_part@partitionid=2 +POSTHOOK: query: insert overwrite table src_stat_part partition (partitionId=2) +select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_stat_part +POSTHOOK: query: describe formatted src_stat_part PARTITION(partitionId=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_stat_part +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +partitionid int + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: src_stat_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 25 + rawDataSize 191 + totalSize 216 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: drop table tab_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tab_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab_part +POSTHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab_part +PREHOOK: query: drop table srcbucket_mapjoin_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srcbucket_mapjoin_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_part +PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_part +POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: describe formatted tab_part partition (ds='2008-04-08') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab_part +POSTHOOK: query: describe formatted tab_part partition (ds='2008-04-08') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab_part +# col_name data_type comment +key int +value string + +# Partition Information +# col_name data_type comment +ds string + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: tab_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 4 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [key] +Sort Columns: [Order(col:key, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab +POSTHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab +PREHOOK: query: insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: query: insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: describe formatted tab partition (ds='2008-04-08') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab +POSTHOOK: query: describe formatted tab partition (ds='2008-04-08') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab +# col_name data_type comment +key int +value string + +# Partition Information +# col_name data_type comment +ds string + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: tab +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 2 + numRows 242 + rawDataSize 2566 + totalSize 2808 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [Order(col:key, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) + partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) + partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: describe formatted nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe formatted nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert overwrite table nzhang_part14 partition(ds, hr) +select key, value, ds, hr from ( + select * from (select 'k1' as key, cast(null as string) as value, '1' as ds, '2' as hr from src limit 2)a + union all + select * from (select 'k2' as key, '' as value, '1' as ds, '3' as hr from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value, '2' as ds, '1' as hr from src limit 2)c +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert overwrite table nzhang_part14 partition(ds, hr) +select key, value, ds, hr from ( + select * from (select 'k1' as key, cast(null as string) as value, '1' as ds, '2' as hr from src limit 2)a + union all + select * from (select 'k2' as key, '' as value, '1' as ds, '3' as hr from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value, '2' as ds, '1' as hr from src limit 2)c +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=2 +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=3 +POSTHOOK: Output: default@nzhang_part14@ds=2/hr=1 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=2).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=2).value EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=3).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=3).value EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2,hr=1).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2,hr=1).value EXPRESSION [] +PREHOOK: query: desc formatted nzhang_part14 partition(ds='1', hr='3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 partition(ds='1', hr='3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Partition Information +Partition Value: [1, 3] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 6 + totalSize 8 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-03-03, 12] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-03-03, 12] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@a +PREHOOK: Output: default@a +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@a +POSTHOOK: Output: default@a +PREHOOK: query: create table a (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@b +PREHOOK: Output: default@b +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@b +POSTHOOK: Output: default@b +PREHOOK: query: create table b (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: drop table c +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table c +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table c (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@c +POSTHOOK: query: create table c (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@c +PREHOOK: query: FROM srcpart +INSERT OVERWRITE TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT OVERWRITE TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT OVERWRITE TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@a@ds=2010-03-11 +PREHOOK: Output: default@b@ds=2010-04-11 +PREHOOK: Output: default@c@ds=2010-05-11 +POSTHOOK: query: FROM srcpart +INSERT OVERWRITE TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT OVERWRITE TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT OVERWRITE TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@a@ds=2010-03-11/hr=11 +POSTHOOK: Output: default@a@ds=2010-03-11/hr=12 +POSTHOOK: Output: default@b@ds=2010-04-11/hr=12 +POSTHOOK: Output: default@c@ds=2010-05-11/hr=11 +POSTHOOK: Output: default@c@ds=2010-05-11/hr=12 +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b PARTITION(ds=2010-04-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b PARTITION(ds=2010-04-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select key from a +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select value from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select key from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select value from c +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: c + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select key from c +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: c + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + diff --git ql/src/test/results/clientpositive/autoColumnStats_10.q.out ql/src/test/results/clientpositive/autoColumnStats_10.q.out new file mode 100644 index 0000000000..6cb51fdab9 --- /dev/null +++ ql/src/test/results/clientpositive/autoColumnStats_10.q.out @@ -0,0 +1,516 @@ +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min 1 +max 1 +num_nulls 0 +distinct_count 1 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min 1 +max 2 +num_nulls 0 +distinct_count 2 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} diff --git ql/src/test/results/clientpositive/autoColumnStats_2.q.out ql/src/test/results/clientpositive/autoColumnStats_2.q.out new file mode 100644 index 0000000000..b209ff0dcc --- /dev/null +++ ql/src/test/results/clientpositive/autoColumnStats_2.q.out @@ -0,0 +1,1557 @@ +PREHOOK: query: drop table src_multi1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_multi1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_multi1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: insert into table src_multi1 select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: insert into table src_multi1 select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Lineage: src_multi1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended select * from src_multi1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select * from src_multi1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_multi1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: describe formatted src_multi1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi1 +POSTHOOK: query: describe formatted src_multi1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi1 +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table a like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: from src +insert into table a select * +insert into table b select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +PREHOOK: Output: default@b +POSTHOOK: query: from src +insert into table a select * +insert into table b select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Output: default@b +POSTHOOK: Lineage: a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted a key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@a +POSTHOOK: query: describe formatted a key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@a +col_name key +data_type string +min +max +num_nulls 0 +distinct_count 309 +avg_col_len 2.812 +max_col_len 3 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: describe formatted b key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name key +data_type string +min +max +num_nulls 0 +distinct_count 309 +avg_col_len 2.812 +max_col_len 3 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: from src +insert overwrite table a select * +insert into table b select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +PREHOOK: Output: default@b +POSTHOOK: query: from src +insert overwrite table a select * +insert into table b select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Output: default@b +POSTHOOK: Lineage: a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@a +POSTHOOK: query: describe formatted a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@a +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted b +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted b key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name key +data_type string +min +max +num_nulls 0 +distinct_count 309 +avg_col_len 2.812 +max_col_len 3 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: describe formatted b value +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b value +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name value +data_type string +min +max +num_nulls 0 +distinct_count 309 +avg_col_len 6.812 +max_col_len 7 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: insert into table b select NULL, NULL from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@b +POSTHOOK: query: insert into table b select NULL, NULL from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@b +POSTHOOK: Lineage: b.key EXPRESSION [] +POSTHOOK: Lineage: b.value EXPRESSION [] +PREHOOK: query: describe formatted b key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name key +data_type string +min +max +num_nulls 10 +distinct_count 309 +avg_col_len 2.812 +max_col_len 3 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: describe formatted b value +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b value +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name value +data_type string +min +max +num_nulls 10 +distinct_count 309 +avg_col_len 6.812 +max_col_len 7 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: insert into table b(value) select key+100000 from src limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@b +POSTHOOK: query: insert into table b(value) select key+100000 from src limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@b +POSTHOOK: Lineage: b.key SIMPLE [] +POSTHOOK: Lineage: b.value EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: describe formatted b key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name key +data_type string +min +max +num_nulls 20 +distinct_count 309 +avg_col_len 2.812 +max_col_len 3 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: describe formatted b value +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b value +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +col_name value +data_type string +min +max +num_nulls 10 +distinct_count 319 +avg_col_len 8.0 +max_col_len 8 +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} +PREHOOK: query: drop table src_multi2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_multi2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_multi2 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: insert into table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: insert into table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_multi2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi2 +POSTHOOK: query: describe formatted src_multi2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi2 +# col_name data_type comment +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 508 + rawDataSize 5400 + totalSize 5908 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table if not exists nzhang_part14 (key string) + partitioned by (value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string) + partitioned by (value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: insert into table nzhang_part14 partition(value) +select key, value from ( + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a + union all + select * from (select 'k2' as key, '' as value from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value from src limit 2)c +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert into table nzhang_part14 partition(value) +select key, value from ( + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a + union all + select * from (select 'k2' as key, '' as value from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value from src limit 2)c +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_part14@value= +POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: nzhang_part14 PARTITION(value= ).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] +PREHOOK: query: explain select key from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 6 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 516 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table src5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src5 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src5 as select key, value from src limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src5 +POSTHOOK: query: create table src5 as select key, value from src limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5 +POSTHOOK: Lineage: src5.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src5.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table nzhang_part14 partition(value) +select key, value from src5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src5 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert into table nzhang_part14 partition(value) +select key, value from src5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src5 +POSTHOOK: Output: default@nzhang_part14@value=val_165 +POSTHOOK: Output: default@nzhang_part14@value=val_238 +POSTHOOK: Output: default@nzhang_part14@value=val_27 +POSTHOOK: Output: default@nzhang_part14@value=val_311 +POSTHOOK: Output: default@nzhang_part14@value=val_86 +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_165).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_238).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_27).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_311).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_86).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: explain select key from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 946 Basic stats: COMPLETE Column stats: PARTIAL + ListSink + +PREHOOK: query: drop table alter5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table alter5 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alter5 +POSTHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter5 +PREHOOK: query: alter table alter5 add partition (dt='a') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@alter5 +POSTHOOK: query: alter table alter5 add partition (dt='a') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@alter5 +POSTHOOK: Output: default@alter5@dt=a +PREHOOK: query: describe formatted alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe formatted alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +# col_name data_type comment +col1 string + +# Partition Information +# col_name data_type comment +dt string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: alter5 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table alter5 partition (dt='a') select key from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@alter5@dt=a +POSTHOOK: query: insert into table alter5 partition (dt='a') select key from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@alter5@dt=a +POSTHOOK: Lineage: alter5 PARTITION(dt=a).col1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: describe formatted alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe formatted alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +# col_name data_type comment +col1 string + +# Partition Information +# col_name data_type comment +dt string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: alter5 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 1406 + totalSize 1906 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select * from alter5 where dt='a' +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from alter5 where dt='a' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: alter5 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col1 (type: string), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 86000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table alter5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@alter5 +PREHOOK: Output: default@alter5 +POSTHOOK: query: drop table alter5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@alter5 +POSTHOOK: Output: default@alter5 +PREHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alter5 +POSTHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter5 +PREHOOK: query: alter table alter5 add partition (dt='a') location 'parta' +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@alter5 +POSTHOOK: query: alter table alter5 add partition (dt='a') location 'parta' +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@alter5 +POSTHOOK: Output: default@alter5@dt=a +PREHOOK: query: describe formatted alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe formatted alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +# col_name data_type comment +col1 string + +# Partition Information +# col_name data_type comment +dt string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: alter5 +#### A masked pattern was here #### +Partition Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table alter5 partition (dt='a') select key from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@alter5@dt=a +POSTHOOK: query: insert into table alter5 partition (dt='a') select key from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@alter5@dt=a +POSTHOOK: Lineage: alter5 PARTITION(dt=a).col1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: describe formatted alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe formatted alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +# col_name data_type comment +col1 string + +# Partition Information +# col_name data_type comment +dt string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: alter5 +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + totalSize 1906 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select * from alter5 where dt='a' +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from alter5 where dt='a' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: alter5 + Statistics: Num rows: 19 Data size: 1653 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col1 (type: string), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 3268 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table src_stat_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_stat_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_stat_part(key string, value string) partitioned by (partitionId int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_stat_part +POSTHOOK: query: create table src_stat_part(key string, value string) partitioned by (partitionId int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_stat_part +PREHOOK: query: insert into table src_stat_part partition (partitionId=1) +select * from src1 limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: query: insert into table src_stat_part partition (partitionId=1) +select * from src1 limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=1).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=1).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_stat_part +POSTHOOK: query: describe formatted src_stat_part PARTITION(partitionId=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_stat_part +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +partitionid int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: src_stat_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 5 + rawDataSize 38 + totalSize 43 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table src_stat_part partition (partitionId=2) +select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_stat_part@partitionid=2 +POSTHOOK: query: insert into table src_stat_part partition (partitionId=2) +select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_stat_part PARTITION(partitionId=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_stat_part +POSTHOOK: query: describe formatted src_stat_part PARTITION(partitionId=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_stat_part +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +partitionid int + +# Detailed Partition Information +Partition Value: [2] +Database: default +Table: src_stat_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 25 + rawDataSize 191 + totalSize 216 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: drop table tab_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tab_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab_part +POSTHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab_part +PREHOOK: query: drop table srcbucket_mapjoin_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srcbucket_mapjoin_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: insert into table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_part +PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: query: insert into table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_part +POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: describe formatted tab_part partition (ds='2008-04-08') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab_part +POSTHOOK: query: describe formatted tab_part partition (ds='2008-04-08') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab_part +# col_name data_type comment +key int +value string + +# Partition Information +# col_name data_type comment +ds string + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: tab_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 4 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [key] +Sort Columns: [Order(col:key, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab +POSTHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab +PREHOOK: query: insert into table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: query: insert into table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: describe formatted tab partition (ds='2008-04-08') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab +POSTHOOK: query: describe formatted tab partition (ds='2008-04-08') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab +# col_name data_type comment +key int +value string + +# Partition Information +# col_name data_type comment +ds string + +# Detailed Partition Information +Partition Value: [2008-04-08] +Database: default +Table: tab +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 2 + numRows 242 + rawDataSize 2566 + totalSize 2808 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 2 +Bucket Columns: [key] +Sort Columns: [Order(col:key, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) + partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) + partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: describe formatted nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe formatted nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into table nzhang_part14 partition(ds, hr) +select key, value, ds, hr from ( + select * from (select 'k1' as key, cast(null as string) as value, '1' as ds, '2' as hr from src limit 2)a + union all + select * from (select 'k2' as key, '' as value, '1' as ds, '3' as hr from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value, '2' as ds, '1' as hr from src limit 2)c +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert into table nzhang_part14 partition(ds, hr) +select key, value, ds, hr from ( + select * from (select 'k1' as key, cast(null as string) as value, '1' as ds, '2' as hr from src limit 2)a + union all + select * from (select 'k2' as key, '' as value, '1' as ds, '3' as hr from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value, '2' as ds, '1' as hr from src limit 2)c +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=2 +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=3 +POSTHOOK: Output: default@nzhang_part14@ds=2/hr=1 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=2).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=2).value EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=3).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=3).value EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2,hr=1).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2,hr=1).value EXPRESSION [] +PREHOOK: query: desc formatted nzhang_part14 partition(ds='1', hr='3') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 partition(ds='1', hr='3') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Partition Information +Partition Value: [1, 3] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 6 + totalSize 8 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: INSERT into TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT into TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-03-03, 12] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: INSERT into TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT into TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 PARTITION(ds='2010-03-03', hr='12') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string +value string + +# Partition Information +# col_name data_type comment +ds string +hr string + +# Detailed Partition Information +Partition Value: [2010-03-03, 12] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@a +PREHOOK: Output: default@a +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@a +POSTHOOK: Output: default@a +PREHOOK: query: create table a (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@b +PREHOOK: Output: default@b +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@b +POSTHOOK: Output: default@b +PREHOOK: query: create table b (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: drop table c +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table c +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table c (key string, value string) +partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@c +POSTHOOK: query: create table c (key string, value string) +partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@c +PREHOOK: query: FROM srcpart +INSERT into TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT into TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT into TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@a@ds=2010-03-11 +PREHOOK: Output: default@b@ds=2010-04-11 +PREHOOK: Output: default@c@ds=2010-05-11 +POSTHOOK: query: FROM srcpart +INSERT into TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT into TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT into TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@a@ds=2010-03-11/hr=11 +POSTHOOK: Output: default@a@ds=2010-03-11/hr=12 +POSTHOOK: Output: default@b@ds=2010-04-11/hr=12 +POSTHOOK: Output: default@c@ds=2010-05-11/hr=11 +POSTHOOK: Output: default@c@ds=2010-05-11/hr=12 +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b PARTITION(ds=2010-04-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b PARTITION(ds=2010-04-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select key from a +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select value from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select key from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select value from c +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: c + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select key from c +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: c + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + diff --git ql/src/test/results/clientpositive/autoColumnStats_3.q.out ql/src/test/results/clientpositive/autoColumnStats_3.q.out index c6527a9879..b3d8d3e71a 100644 --- ql/src/test/results/clientpositive/autoColumnStats_3.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_3.q.out @@ -13,10 +13,12 @@ POSTHOOK: Output: default@src_multi1 PREHOOK: query: analyze table src_multi1 compute statistics for columns key PREHOOK: type: QUERY PREHOOK: Input: default@src_multi1 +PREHOOK: Output: default@src_multi1 #### A masked pattern was here #### POSTHOOK: query: analyze table src_multi1 compute statistics for columns key POSTHOOK: type: QUERY POSTHOOK: Input: default@src_multi1 +POSTHOOK: Output: default@src_multi1 #### A masked pattern was here #### PREHOOK: query: describe formatted src_multi1 PREHOOK: type: DESCTABLE @@ -222,11 +224,15 @@ PREHOOK: query: analyze table nzhang_part14 partition(ds='1', hr='3') compute st PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_part14 PREHOOK: Input: default@nzhang_part14@ds=1/hr=3 +PREHOOK: Output: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14@ds=1/hr=3 #### A masked pattern was here #### POSTHOOK: query: analyze table nzhang_part14 partition(ds='1', hr='3') compute statistics for columns value POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_part14 POSTHOOK: Input: default@nzhang_part14@ds=1/hr=3 +POSTHOOK: Output: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=3 #### A masked pattern was here #### PREHOOK: query: desc formatted nzhang_part14 partition(ds='1', hr='3') PREHOOK: type: DESCTABLE @@ -355,7 +361,7 @@ Database: default Table: nzhang_part14 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} numFiles 2 numRows 4 rawDataSize 12 @@ -393,7 +399,7 @@ Database: default Table: nzhang_part14 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} numFiles 2 numRows 4 rawDataSize 16 diff --git ql/src/test/results/clientpositive/autoColumnStats_4.q.out ql/src/test/results/clientpositive/autoColumnStats_4.q.out index 9155733e7c..3f169e4f80 100644 --- ql/src/test/results/clientpositive/autoColumnStats_4.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_4.q.out @@ -49,8 +49,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - Stage-5 depends on stages: Stage-3, Stage-4 + Stage-3 depends on stages: Stage-0, Stage-4 Stage-4 depends on stages: Stage-2 STAGE PLANS: @@ -138,10 +137,8 @@ STAGE PLANS: Write Type: INSERT Stage: Stage-3 - Stats-Aggr Operator - - Stage: Stage-5 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b Column Types: int, varchar(128) @@ -196,10 +193,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} numFiles 2 - numRows 0 - rawDataSize 0 totalSize 1798 transactional true #### A masked pattern was here #### @@ -239,9 +233,8 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {} numFiles 4 - numRows 0 - rawDataSize 0 totalSize 2909 transactional true #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/autoColumnStats_5.q.out ql/src/test/results/clientpositive/autoColumnStats_5.q.out index cc4d32b5fd..196d18d851 100644 --- ql/src/test/results/clientpositive/autoColumnStats_5.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_5.q.out @@ -17,7 +17,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -98,10 +97,8 @@ STAGE PLANS: name: default.partitioned1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b Column Types: int, string @@ -260,7 +257,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -341,10 +337,8 @@ STAGE PLANS: name: default.partitioned1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d Column Types: int, string, int, string @@ -461,7 +455,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -542,10 +535,8 @@ STAGE PLANS: name: default.partitioned1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d Column Types: int, string, int, string @@ -613,7 +604,7 @@ Database: default Table: partitioned1 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\",\"d\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} numFiles 2 numRows 6 rawDataSize 78 @@ -656,15 +647,5 @@ POSTHOOK: query: desc formatted partitioned1 partition(part=1) c POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@partitioned1 col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses bitvector comment -col_name c -data_type int -min 100 -max 200 -num_nulls 0 -distinct_count 2 -avg_col_len -max_col_len -num_trues -num_falses -bitVector HL -comment from deserializer +# col_name data_type comment +c int from deserializer diff --git ql/src/test/results/clientpositive/autoColumnStats_5a.q.out ql/src/test/results/clientpositive/autoColumnStats_5a.q.out new file mode 100644 index 0000000000..d97e1c6904 --- /dev/null +++ ql/src/test/results/clientpositive/autoColumnStats_5a.q.out @@ -0,0 +1,997 @@ +PREHOOK: query: CREATE TABLE partitioned1(a INT, b STRING) PARTITIONED BY(part INT) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@partitioned1 +POSTHOOK: query: CREATE TABLE partitioned1(a INT, b STRING) PARTITIONED BY(part INT) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@partitioned1 +PREHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(1, 'original') +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(1, 'original') +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__1 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: part=1/ + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), UDFToInteger('1') (type: int) + outputColumnNames: a, b, part + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + keys: part (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: struct), _col2 (type: struct) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: Values__Tmp__Table__1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__1 + serialization.ddl struct values__tmp__table__1 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.values__tmp__table__1 + name: default.values__tmp__table__1 + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 11 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types struct:struct:int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + part 1 + replace: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-2 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + Column Stats Desc: + Columns: a, b + Column Types: int, string + Table: default.partitioned1 + Is Table Level Stats: false + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert into table partitioned1 partition(part=1) values(1, 'original') +PREHOOK: type: QUERY +PREHOOK: Output: default@partitioned1@part=1 +POSTHOOK: query: insert into table partitioned1 partition(part=1) values(1, 'original') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@partitioned1@part=1 +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).a EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).b SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +_col0 _col1 +PREHOOK: query: desc formatted partitioned1 partition(part=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 partition(part=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type comment +# col_name data_type comment +a int +b string + +# Partition Information +# col_name data_type comment +part int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: partitioned1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 10 + totalSize 11 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__3 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Static Partition Specification: part=1/ + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string), UDFToInteger('1') (type: int) + outputColumnNames: a, b, part + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + keys: part (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: struct), _col2 (type: struct) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: Values__Tmp__Table__3 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__3 + serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns tmp_values_col1,tmp_values_col2 + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.values__tmp__table__3 + serialization.ddl struct values__tmp__table__3 { string tmp_values_col1, string tmp_values_col2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.values__tmp__table__3 + name: default.values__tmp__table__3 + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types struct:struct:int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + part 1 + replace: false +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-2 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + Column Stats Desc: + Columns: a, b + Column Types: int, string + Table: default.partitioned1 + Is Table Level Stats: false + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns part + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -ext-10002 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns a,b + columns.comments + columns.types int:string +#### A masked pattern was here #### + name default.partitioned1 + partition_columns.types int + serialization.ddl struct partitioned1 { i32 a, string b} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + name: default.partitioned1 + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +PREHOOK: type: QUERY +PREHOOK: Output: default@partitioned1@part=1 +POSTHOOK: query: insert into table partitioned1 partition(part=1) values(2, 'original'), (3, 'original'),(4, 'original') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@partitioned1@part=1 +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).a EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: partitioned1 PARTITION(part=1).b SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +_col0 _col1 +PREHOOK: query: explain insert into table partitioned1 partition(part=1) values(1, 'original'),(2, 'original'), (3, 'original'),(4, 'original') +PREHOOK: type: QUERY +POSTHOOK: query: explain insert into table partitioned1 partition(part=1) values(1, 'original'),(2, 'original'), (3, 'original'),(4, 'original') +POSTHOOK: type: QUERY +Explain +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: values__tmp__table__5 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: UDFToInteger(tmp_values_col1) (type: int), tmp_values_col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + Select Operator + expressions: _col0 (type: int), _col1 (type: string), UDFToInteger('1') (type: int) + outputColumnNames: a, b, part + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(a, 'hll'), compute_stats(b, 'hll') + keys: part (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct), _col2 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + part 1 + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: a, b + Column Types: int, string + Table: default.partitioned1 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.partitioned1 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: desc formatted partitioned1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type comment +# col_name data_type comment +a int +b string + +# Partition Information +# col_name data_type comment +part int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numPartitions 1 + numRows 4 + rawDataSize 40 + totalSize 44 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted partitioned1 partition(part=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 partition(part=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type comment +# col_name data_type comment +a int +b string + +# Partition Information +# col_name data_type comment +part int + +# Detailed Partition Information +Partition Value: [1] +Database: default +Table: partitioned1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}} + numFiles 2 + numRows 4 + rawDataSize 40 + totalSize 44 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted partitioned1 partition(part=1) a +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@partitioned1 +POSTHOOK: query: desc formatted partitioned1 partition(part=1) a +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@partitioned1 +col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses bitvector comment +col_name a +data_type int +min 1 +max 4 +num_nulls 0 +distinct_count 4 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer diff --git ql/src/test/results/clientpositive/autoColumnStats_6.q.out ql/src/test/results/clientpositive/autoColumnStats_6.q.out index c4ab489b39..70788fde9f 100644 --- ql/src/test/results/clientpositive/autoColumnStats_6.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_6.q.out @@ -30,7 +30,6 @@ STAGE DEPENDENCIES: Stage-4 Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2 Stage-3 Stage-5 Stage-6 depends on stages: Stage-5 @@ -113,10 +112,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string diff --git ql/src/test/results/clientpositive/autoColumnStats_7.q.out ql/src/test/results/clientpositive/autoColumnStats_7.q.out index 2dc9fc2d42..acea69e484 100644 --- ql/src/test/results/clientpositive/autoColumnStats_7.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_7.q.out @@ -26,8 +26,7 @@ STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-3, Stage-5 + Stage-3 depends on stages: Stage-0, Stage-5 Stage-4 depends on stages: Stage-2 Stage-5 depends on stages: Stage-4 @@ -112,10 +111,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator - - Stage: Stage-6 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, c1, c2 Column Types: string, int, string diff --git ql/src/test/results/clientpositive/autoColumnStats_8.q.out ql/src/test/results/clientpositive/autoColumnStats_8.q.out index 9d22aebc0d..1d7e966a31 100644 --- ql/src/test/results/clientpositive/autoColumnStats_8.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_8.q.out @@ -47,10 +47,8 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-3, Stage-4, Stage-5 - Stage-7 depends on stages: Stage-3, Stage-4, Stage-5 Stage-1 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-1 + Stage-4 depends on stages: Stage-1, Stage-5 Stage-5 depends on stages: Stage-2 STAGE PLANS: @@ -447,25 +445,10 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### - Stage: Stage-6 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.nzhang_part8 - Is Table Level Stats: false - - Stage: Stage-7 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.nzhang_part8 - Is Table Level Stats: false - Stage: Stage-1 Move Operator tables: @@ -495,8 +478,14 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.nzhang_part8 + Is Table Level Stats: false Stage: Stage-5 Map Reduce diff --git ql/src/test/results/clientpositive/autoColumnStats_9.q.out ql/src/test/results/clientpositive/autoColumnStats_9.q.out index c6d12fa440..0dfcf048f2 100644 --- ql/src/test/results/clientpositive/autoColumnStats_9.q.out +++ ql/src/test/results/clientpositive/autoColumnStats_9.q.out @@ -20,8 +20,7 @@ STAGE DEPENDENCIES: Stage-7 Stage-5 depends on stages: Stage-7 Stage-0 depends on stages: Stage-5 - Stage-2 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-2, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-3 Stage-3 depends on stages: Stage-5 STAGE PLANS: @@ -166,10 +165,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string diff --git ql/src/test/results/clientpositive/auto_join1.q.out ql/src/test/results/clientpositive/auto_join1.q.out index 5f4bb7452a..c2d24739ab 100644 --- ql/src/test/results/clientpositive/auto_join1.q.out +++ ql/src/test/results/clientpositive/auto_join1.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/auto_join14.q.out ql/src/test/results/clientpositive/auto_join14.q.out index 1dd677c3d6..e67506b3b8 100644 --- ql/src/test/results/clientpositive/auto_join14.q.out +++ ql/src/test/results/clientpositive/auto_join14.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/auto_join17.q.out ql/src/test/results/clientpositive/auto_join17.q.out index d39c36eac3..0239cf8285 100644 --- ql/src/test/results/clientpositive/auto_join17.q.out +++ ql/src/test/results/clientpositive/auto_join17.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* diff --git ql/src/test/results/clientpositive/auto_join19.q.out ql/src/test/results/clientpositive/auto_join19.q.out index 3f70055d81..e70c5147fb 100644 --- ql/src/test/results/clientpositive/auto_join19.q.out +++ ql/src/test/results/clientpositive/auto_join19.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/auto_join19_inclause.q.out ql/src/test/results/clientpositive/auto_join19_inclause.q.out index 3f70055d81..e70c5147fb 100644 --- ql/src/test/results/clientpositive/auto_join19_inclause.q.out +++ ql/src/test/results/clientpositive/auto_join19_inclause.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/auto_join2.q.out ql/src/test/results/clientpositive/auto_join2.q.out index b17d344985..4132caf7fe 100644 --- ql/src/test/results/clientpositive/auto_join2.q.out +++ ql/src/test/results/clientpositive/auto_join2.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/auto_join26.q.out ql/src/test/results/clientpositive/auto_join26.q.out index e6d966ffad..9b457415e1 100644 --- ql/src/test/results/clientpositive/auto_join26.q.out +++ ql/src/test/results/clientpositive/auto_join26.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/auto_join3.q.out ql/src/test/results/clientpositive/auto_join3.q.out index 35e8273766..5921da093c 100644 --- ql/src/test/results/clientpositive/auto_join3.q.out +++ ql/src/test/results/clientpositive/auto_join3.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/auto_join4.q.out ql/src/test/results/clientpositive/auto_join4.q.out index 3c0ab85b05..fc34040d24 100644 --- ql/src/test/results/clientpositive/auto_join4.q.out +++ ql/src/test/results/clientpositive/auto_join4.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join5.q.out ql/src/test/results/clientpositive/auto_join5.q.out index 8da15a55d3..fe4e24e80d 100644 --- ql/src/test/results/clientpositive/auto_join5.q.out +++ ql/src/test/results/clientpositive/auto_join5.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join6.q.out ql/src/test/results/clientpositive/auto_join6.q.out index 00bbb1bb21..594e7a44e4 100644 --- ql/src/test/results/clientpositive/auto_join6.q.out +++ ql/src/test/results/clientpositive/auto_join6.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join7.q.out ql/src/test/results/clientpositive/auto_join7.q.out index 339ea70a21..13a5ff507d 100644 --- ql/src/test/results/clientpositive/auto_join7.q.out +++ ql/src/test/results/clientpositive/auto_join7.q.out @@ -138,7 +138,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join8.q.out ql/src/test/results/clientpositive/auto_join8.q.out index 4cc3d598f7..704ce3c31d 100644 --- ql/src/test/results/clientpositive/auto_join8.q.out +++ ql/src/test/results/clientpositive/auto_join8.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/auto_join9.q.out ql/src/test/results/clientpositive/auto_join9.q.out index d7d7d181f0..e5022a3d29 100644 --- ql/src/test/results/clientpositive/auto_join9.q.out +++ ql/src/test/results/clientpositive/auto_join9.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12' diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out index d8479376eb..f647b9fc31 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out @@ -161,10 +161,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -210,10 +208,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -358,10 +354,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -407,10 +401,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -532,10 +524,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -638,10 +628,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -687,10 +675,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -735,10 +721,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -821,10 +805,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -869,10 +851,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -975,10 +955,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1024,10 +1002,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1072,10 +1048,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1187,10 +1161,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1236,10 +1208,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out index 243a49b45f..95812b936c 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_11.q.out @@ -130,10 +130,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -238,10 +236,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -286,10 +282,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -334,10 +328,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -454,10 +446,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -562,10 +552,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -610,10 +598,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -658,10 +644,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -777,10 +761,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -885,10 +867,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -933,10 +913,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1052,10 +1030,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1102,10 +1078,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1148,10 +1122,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1275,10 +1247,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1323,10 +1293,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out index 7875e9693a..5c4f377a13 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out @@ -168,10 +168,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -220,10 +218,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -272,10 +268,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -423,10 +417,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -472,10 +464,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -521,10 +511,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -570,10 +558,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out index 8c7658c447..486d7b3d35 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_13.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -139,7 +140,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -303,7 +305,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -316,7 +319,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -480,7 +484,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -493,7 +498,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out index 4c31a693ac..f9dd101edf 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_2.q.out @@ -143,10 +143,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -192,10 +190,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -317,10 +313,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -423,10 +417,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -472,10 +464,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -520,10 +510,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -606,10 +594,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -654,10 +640,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -760,10 +744,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -809,10 +791,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -857,10 +837,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -972,10 +950,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1021,10 +997,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out index caa30175b9..cea6003277 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_3.q.out @@ -143,10 +143,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -290,10 +288,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -414,10 +410,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -461,10 +455,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -567,10 +559,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -615,10 +605,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -663,10 +651,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -748,10 +734,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -854,10 +838,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -902,10 +884,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -950,10 +930,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1066,10 +1044,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out index 21a1175257..e3d7c08277 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_4.q.out @@ -159,10 +159,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -306,10 +304,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -430,10 +426,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -477,10 +471,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -583,10 +575,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -631,10 +621,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -679,10 +667,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -764,10 +750,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -870,10 +854,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -918,10 +900,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -966,10 +946,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1082,10 +1060,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out index 0e6bbf1aad..0cb2cee412 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_5.q.out @@ -76,16 +76,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -123,8 +123,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -145,8 +143,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -216,16 +212,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -263,8 +259,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -285,8 +279,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -369,16 +361,16 @@ STAGE PLANS: $hdt$_1:b TableScan alias: b - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col0 (type: string) @@ -390,16 +382,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -438,8 +430,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -460,8 +450,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -501,8 +489,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -551,16 +537,16 @@ STAGE PLANS: $hdt$_0:a TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col0 (type: string) @@ -572,16 +558,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -620,8 +606,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -642,8 +626,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -683,8 +665,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -728,16 +708,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -775,8 +755,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -797,8 +775,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out index a4cdbec0f2..649fe52bd8 100644 --- ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/auto_sortmerge_join_7.q.out @@ -176,10 +176,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -225,10 +223,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -375,10 +371,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -424,10 +418,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -551,10 +543,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -598,10 +588,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -704,10 +692,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -753,10 +739,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -801,10 +785,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -849,10 +831,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -935,10 +915,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -983,10 +961,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1089,10 +1065,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1138,10 +1112,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1186,10 +1158,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1234,10 +1204,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1350,10 +1318,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1399,10 +1365,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/avro_decimal.q.out ql/src/test/results/clientpositive/avro_decimal.q.out index b093a22aa5..4d69221851 100644 --- ql/src/test/results/clientpositive/avro_decimal.q.out +++ ql/src/test/results/clientpositive/avro_decimal.q.out @@ -21,10 +21,12 @@ POSTHOOK: Output: default@dec PREHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value PREHOOK: type: QUERY PREHOOK: Input: default@dec +PREHOOK: Output: default@dec #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value POSTHOOK: type: QUERY POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec #### A masked pattern was here #### PREHOOK: query: DESC FORMATTED `dec` value PREHOOK: type: DESCTABLE @@ -44,7 +46,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"value\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/avro_decimal_native.q.out ql/src/test/results/clientpositive/avro_decimal_native.q.out index abf5937bc5..e4b1739150 100644 --- ql/src/test/results/clientpositive/avro_decimal_native.q.out +++ ql/src/test/results/clientpositive/avro_decimal_native.q.out @@ -25,10 +25,12 @@ POSTHOOK: Output: default@dec PREHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value PREHOOK: type: QUERY PREHOOK: Input: default@dec +PREHOOK: Output: default@dec #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value POSTHOOK: type: QUERY POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec #### A masked pattern was here #### PREHOOK: query: DESC FORMATTED `dec` value PREHOOK: type: DESCTABLE @@ -48,7 +50,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"value\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/basicstat_partval.q.out ql/src/test/results/clientpositive/basicstat_partval.q.out new file mode 100644 index 0000000000..fa5c5baf8e --- /dev/null +++ ql/src/test/results/clientpositive/basicstat_partval.q.out @@ -0,0 +1,132 @@ +PREHOOK: query: CREATE TABLE p1(i int) partitioned by (p string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p1 +POSTHOOK: query: CREATE TABLE p1(i int) partitioned by (p string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p1 +PREHOOK: query: insert into p1 partition(p='a') values (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@p1@p=a +POSTHOOK: query: insert into p1 partition(p='a') values (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p1@p=a +POSTHOOK: Lineage: p1 PARTITION(p=a).i EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: insert into p1 partition(p='A') values (2),(3) +PREHOOK: type: QUERY +PREHOOK: Output: default@p1@p=A +POSTHOOK: query: insert into p1 partition(p='A') values (2),(3) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p1@p=A +POSTHOOK: Lineage: p1 PARTITION(p=A).i EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: describe formatted p1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p1 +POSTHOOK: query: describe formatted p1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p1 +# col_name data_type comment +i int + +# Partition Information +# col_name data_type comment +p string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numPartitions 2 + numRows 3 + rawDataSize 3 + totalSize 6 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted p1 partition(p='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p1 +POSTHOOK: query: describe formatted p1 partition(p='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p1 +# col_name data_type comment +i int + +# Partition Information +# col_name data_type comment +p string + +# Detailed Partition Information +Partition Value: [a] +Database: default +Table: p1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 1 + totalSize 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted p1 partition(p='A') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p1 +POSTHOOK: query: describe formatted p1 partition(p='A') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p1 +# col_name data_type comment +i int + +# Partition Information +# col_name data_type comment +p string + +# Detailed Partition Information +Partition Value: [A] +Database: default +Table: p1 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 2 + rawDataSize 2 + totalSize 4 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git ql/src/test/results/clientpositive/beeline/colstats_all_nulls.q.out ql/src/test/results/clientpositive/beeline/colstats_all_nulls.q.out index 7e26ae0f06..49af8f6bc4 100644 --- ql/src/test/results/clientpositive/beeline/colstats_all_nulls.q.out +++ ql/src/test/results/clientpositive/beeline/colstats_all_nulls.q.out @@ -31,10 +31,12 @@ PREHOOK: query: analyze table all_nulls compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@all_nulls #### A masked pattern was here #### +PREHOOK: Output: default@all_nulls POSTHOOK: query: analyze table all_nulls compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@all_nulls #### A masked pattern was here #### +POSTHOOK: Output: default@all_nulls PREHOOK: query: describe formatted all_nulls a PREHOOK: type: DESCTABLE PREHOOK: Input: default@all_nulls diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out index b3f36ad580..c37941e626 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_1.q.out @@ -46,6 +46,49 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: desc formatted smb_bucket_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@smb_bucket_1 +POSTHOOK: query: desc formatted smb_bucket_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@smb_bucket_1 + NULL NULL + NULL NULL + SORTBUCKETCOLSPREFIX TRUE + numFiles 1 + serialization.format 1 + totalSize 208 +#### A masked pattern was here #### +# Detailed Table Information NULL NULL +# Storage Information NULL NULL +# col_name data_type comment +Bucket Columns: [key] NULL +Compressed: No NULL +#### A masked pattern was here #### +Database: test_db_smb_mapjoin_1 NULL +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat NULL +LastAccessTime: UNKNOWN NULL +#### A masked pattern was here #### +Num Buckets: 1 NULL +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat NULL +#### A masked pattern was here #### +Retention: 0 NULL +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe NULL +Sort Columns: [Order(col:key, order:1)] NULL +Storage Desc Params: NULL NULL +Table Parameters: NULL NULL +Table Type: MANAGED_TABLE NULL +key int +value string +PREHOOK: query: select count(*) from smb_bucket_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from smb_bucket_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +#### A masked pattern was here #### +5 PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key PREHOOK: type: QUERY @@ -283,10 +326,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -336,7 +379,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Left Outer Join 0 to 1 @@ -391,7 +434,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Right Outer Join 0 to 1 @@ -445,7 +488,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Outer Join 0 to 1 diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out index b53e6704cc..b76e7a8221 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_11.q.out @@ -202,7 +202,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out index 9af4683451..6def3de813 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_12.q.out @@ -204,7 +204,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' @@ -409,7 +410,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') diff --git ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out index 82f5804eea..a2a8660c3b 100644 --- ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out +++ ql/src/test/results/clientpositive/beeline/smb_mapjoin_7.q.out @@ -669,7 +669,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/binary_output_format.q.out ql/src/test/results/clientpositive/binary_output_format.q.out index ddb6adf4a0..ce85351c91 100644 --- ql/src/test/results/clientpositive/binary_output_format.q.out +++ ql/src/test/results/clientpositive/binary_output_format.q.out @@ -208,7 +208,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/bucket1.q.out ql/src/test/results/clientpositive/bucket1.q.out index 1d204731cd..c59a755428 100644 --- ql/src/test/results/clientpositive/bucket1.q.out +++ ql/src/test/results/clientpositive/bucket1.q.out @@ -160,7 +160,8 @@ STAGE PLANS: name: default.bucket1_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket1_1 diff --git ql/src/test/results/clientpositive/bucket2.q.out ql/src/test/results/clientpositive/bucket2.q.out index 48ccafb746..4bee31b05c 100644 --- ql/src/test/results/clientpositive/bucket2.q.out +++ ql/src/test/results/clientpositive/bucket2.q.out @@ -160,7 +160,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/bucket3.q.out ql/src/test/results/clientpositive/bucket3.q.out index b1173e7b35..49f12f7fe9 100644 --- ql/src/test/results/clientpositive/bucket3.q.out +++ ql/src/test/results/clientpositive/bucket3.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') diff --git ql/src/test/results/clientpositive/bucket_map_join_1.q.out ql/src/test/results/clientpositive/bucket_map_join_1.q.out index 63107dd0bf..a9cc25939b 100644 --- ql/src/test/results/clientpositive/bucket_map_join_1.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_1.q.out @@ -128,8 +128,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -150,8 +148,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucket_map_join_2.q.out ql/src/test/results/clientpositive/bucket_map_join_2.q.out index c61cdd9401..a3e10cf921 100644 --- ql/src/test/results/clientpositive/bucket_map_join_2.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_2.q.out @@ -128,8 +128,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -150,8 +148,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out index 70cd53c6e5..d12ca7cfc9 100644 --- ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_spark1.q.out @@ -146,10 +146,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -279,10 +277,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -327,10 +323,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -390,7 +384,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -480,10 +475,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -613,10 +606,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -661,10 +652,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -724,7 +713,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out index d0c3a1aca0..3246db644f 100644 --- ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_spark2.q.out @@ -130,10 +130,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -263,10 +261,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -311,10 +307,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -374,7 +368,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -464,10 +459,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -597,10 +590,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -645,10 +636,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -708,7 +697,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out index eed4a5a970..163b4e204d 100644 --- ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out +++ ql/src/test/results/clientpositive/bucket_map_join_spark3.q.out @@ -130,10 +130,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -263,10 +261,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -311,10 +307,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -374,7 +368,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -464,10 +459,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -597,10 +590,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -645,10 +636,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -708,7 +697,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/bucketcontext_1.q.out ql/src/test/results/clientpositive/bucketcontext_1.q.out index 966cef7ede..1cf76f3cb6 100644 --- ql/src/test/results/clientpositive/bucketcontext_1.q.out +++ ql/src/test/results/clientpositive/bucketcontext_1.q.out @@ -129,10 +129,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -238,10 +236,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -287,10 +283,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -433,10 +427,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -482,10 +474,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_2.q.out ql/src/test/results/clientpositive/bucketcontext_2.q.out index 6a6703ad4b..06e98e2f57 100644 --- ql/src/test/results/clientpositive/bucketcontext_2.q.out +++ ql/src/test/results/clientpositive/bucketcontext_2.q.out @@ -113,10 +113,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -222,10 +220,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -271,10 +267,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -417,10 +411,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -466,10 +458,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_3.q.out ql/src/test/results/clientpositive/bucketcontext_3.q.out index d198a29104..0e9f2af8f3 100644 --- ql/src/test/results/clientpositive/bucketcontext_3.q.out +++ ql/src/test/results/clientpositive/bucketcontext_3.q.out @@ -113,10 +113,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -160,10 +158,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -269,10 +265,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -414,10 +408,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_4.q.out ql/src/test/results/clientpositive/bucketcontext_4.q.out index fd9693c951..3a4b26eeb2 100644 --- ql/src/test/results/clientpositive/bucketcontext_4.q.out +++ ql/src/test/results/clientpositive/bucketcontext_4.q.out @@ -129,10 +129,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -176,10 +174,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -285,10 +281,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -430,10 +424,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_5.q.out ql/src/test/results/clientpositive/bucketcontext_5.q.out index 1326629eb3..629015f527 100644 --- ql/src/test/results/clientpositive/bucketcontext_5.q.out +++ ql/src/test/results/clientpositive/bucketcontext_5.q.out @@ -82,12 +82,12 @@ STAGE PLANS: a TableScan alias: a - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string) @@ -106,12 +106,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -119,7 +119,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Group By Operator aggregations: count() @@ -154,8 +154,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -176,8 +174,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -249,12 +245,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -292,8 +288,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -314,8 +308,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_6.q.out ql/src/test/results/clientpositive/bucketcontext_6.q.out index 069a895f3f..e730721c68 100644 --- ql/src/test/results/clientpositive/bucketcontext_6.q.out +++ ql/src/test/results/clientpositive/bucketcontext_6.q.out @@ -100,12 +100,12 @@ STAGE PLANS: a TableScan alias: a - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string) @@ -137,7 +137,7 @@ STAGE PLANS: 0 key (type: string) 1 key (type: string) Position of Big Table: 1 - Statistics: Num rows: 59 Data size: 6050 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 63 Data size: 6393 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Group By Operator aggregations: count() @@ -173,10 +173,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -222,10 +220,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -366,10 +362,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -415,10 +409,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_7.q.out ql/src/test/results/clientpositive/bucketcontext_7.q.out index a7435125d4..a5a92afcae 100644 --- ql/src/test/results/clientpositive/bucketcontext_7.q.out +++ ql/src/test/results/clientpositive/bucketcontext_7.q.out @@ -146,10 +146,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -193,10 +191,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -302,10 +298,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -351,10 +345,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -499,10 +491,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -548,10 +538,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketcontext_8.q.out ql/src/test/results/clientpositive/bucketcontext_8.q.out index e262907def..8ba7c2089d 100644 --- ql/src/test/results/clientpositive/bucketcontext_8.q.out +++ ql/src/test/results/clientpositive/bucketcontext_8.q.out @@ -146,10 +146,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -193,10 +191,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -302,10 +298,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -351,10 +345,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -499,10 +491,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -548,10 +538,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketmapjoin10.q.out ql/src/test/results/clientpositive/bucketmapjoin10.q.out index 6a8dd806c9..6ac2c3c455 100644 --- ql/src/test/results/clientpositive/bucketmapjoin10.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin10.q.out @@ -164,10 +164,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 3 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -210,10 +208,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -310,10 +306,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -358,10 +352,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 3 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketmapjoin11.q.out ql/src/test/results/clientpositive/bucketmapjoin11.q.out index dabb018be9..01f66b942b 100644 --- ql/src/test/results/clientpositive/bucketmapjoin11.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin11.q.out @@ -172,10 +172,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -218,10 +216,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -326,10 +322,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -374,10 +368,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -505,10 +497,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -551,10 +541,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -659,10 +647,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -707,10 +693,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketmapjoin12.q.out ql/src/test/results/clientpositive/bucketmapjoin12.q.out index 32927b570f..0386b954a4 100644 --- ql/src/test/results/clientpositive/bucketmapjoin12.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin12.q.out @@ -133,10 +133,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -240,10 +238,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -365,10 +361,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_3 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_3 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -465,10 +459,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketmapjoin5.q.out ql/src/test/results/clientpositive/bucketmapjoin5.q.out index 5657cbf870..a711dcbcd0 100644 --- ql/src/test/results/clientpositive/bucketmapjoin5.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin5.q.out @@ -196,12 +196,12 @@ STAGE PLANS: a TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: int) @@ -294,10 +294,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -342,10 +340,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -415,7 +411,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -737,12 +734,12 @@ STAGE PLANS: a TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: int) @@ -835,10 +832,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -883,10 +878,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -956,7 +949,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 diff --git ql/src/test/results/clientpositive/bucketmapjoin8.q.out ql/src/test/results/clientpositive/bucketmapjoin8.q.out index 098578534a..c3a73be2e6 100644 --- ql/src/test/results/clientpositive/bucketmapjoin8.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin8.q.out @@ -98,10 +98,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -206,10 +204,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -340,10 +336,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -448,10 +442,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketmapjoin9.q.out ql/src/test/results/clientpositive/bucketmapjoin9.q.out index 1586557789..c2767aecec 100644 --- ql/src/test/results/clientpositive/bucketmapjoin9.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin9.q.out @@ -106,10 +106,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 3 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -206,10 +204,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -373,10 +369,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -473,10 +467,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out index 2da33c6961..b05c12432e 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative.q.out @@ -109,10 +109,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -161,12 +159,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -232,8 +230,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -253,8 +249,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -304,7 +298,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out index bf1c0118cf..57826a3354 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative2.q.out @@ -118,10 +118,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -164,10 +162,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -223,12 +219,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -295,8 +291,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -316,8 +310,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -367,7 +359,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out index ece8f765e7..fae47c8fab 100644 --- ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out +++ ql/src/test/results/clientpositive/bucketmapjoin_negative3.q.out @@ -162,12 +162,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -186,12 +186,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -200,18 +200,18 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -249,8 +249,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -271,8 +269,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -310,12 +306,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -334,12 +330,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -348,18 +344,18 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -397,8 +393,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -419,8 +413,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -458,12 +450,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: UDFToDouble(key) is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 (key + key) (type: double) @@ -475,12 +467,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key + key) is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -489,17 +481,17 @@ STAGE PLANS: 1 UDFToDouble(key) (type: double) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -537,8 +529,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -559,8 +549,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -598,12 +586,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -615,12 +603,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -629,17 +617,17 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -677,8 +665,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -699,8 +685,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -738,12 +722,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -755,12 +739,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -769,17 +753,17 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -817,8 +801,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -839,8 +821,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -878,12 +858,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -895,12 +875,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -909,17 +889,17 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -957,8 +937,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -979,8 +957,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1018,12 +994,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1035,12 +1011,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1049,17 +1025,17 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1097,8 +1073,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1119,8 +1093,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1158,12 +1130,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1175,12 +1147,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1189,17 +1161,17 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1237,8 +1209,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1259,8 +1229,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1298,12 +1266,12 @@ STAGE PLANS: r TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1315,12 +1283,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1329,17 +1297,17 @@ STAGE PLANS: 1 key (type: string), value (type: string) outputColumnNames: _col0, _col1, _col5, _col6 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1377,8 +1345,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1399,8 +1365,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out index 165f0dc1e5..2d6bd6f709 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT x.key, x.value from @@ -187,7 +188,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT * from @@ -292,7 +294,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') @@ -357,7 +360,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') @@ -411,5 +415,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out index c5e03be100..7eb36bfe33 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT x.value, x.key from @@ -196,7 +197,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT x.key, x.value from diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out index 1d794c3d28..7efb7cef02 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work @@ -427,7 +428,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out index 1e70105b9b..6c10249dea 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work @@ -437,7 +438,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out index 1b3d741206..7f56cf3836 100644 --- ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out +++ ql/src/test/results/clientpositive/bucketsortoptimize_insert_8.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work @@ -435,7 +436,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/case_sensitivity.q.out ql/src/test/results/clientpositive/case_sensitivity.q.out index b3969ccf90..c2f2f8b39f 100644 --- ql/src/test/results/clientpositive/case_sensitivity.q.out +++ ql/src/test/results/clientpositive/case_sensitivity.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/cast1.q.out ql/src/test/results/clientpositive/cast1.q.out index 9feb14f1bb..a0cd142bb9 100644 --- ql/src/test/results/clientpositive/cast1.q.out +++ ql/src/test/results/clientpositive/cast1.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out index a4c9a6a15f..eac491eeb2 100644 --- ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out @@ -76,10 +76,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select a, c, min(b) from ( select state as a, locid as b, count(*) as c @@ -177,10 +179,12 @@ STAGE PLANS: PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select year from loc_orc group by year PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out index afc4fd90f9..7fddda8d73 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join1.q.out @@ -49,10 +49,12 @@ POSTHOOK: Output: default@tbl1 PREHOOK: query: analyze table tbl1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tbl1 +PREHOOK: Output: default@tbl1 #### A masked pattern was here #### POSTHOOK: query: analyze table tbl1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl1 +POSTHOOK: Output: default@tbl1 #### A masked pattern was here #### PREHOOK: query: analyze table tbl2 compute statistics PREHOOK: type: QUERY @@ -65,10 +67,12 @@ POSTHOOK: Output: default@tbl2 PREHOOK: query: analyze table tbl2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tbl2 +PREHOOK: Output: default@tbl2 #### A masked pattern was here #### POSTHOOK: query: analyze table tbl2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tbl2 +POSTHOOK: Output: default@tbl2 #### A masked pattern was here #### PREHOOK: query: explain select count(*) from ( diff --git ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out index b29628038c..d3747afd9a 100644 --- ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out +++ ql/src/test/results/clientpositive/cbo_rp_auto_join17.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* diff --git ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out index d4d70bc35e..e5d4421ba7 100644 --- ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/cbo_rp_gby2_map_multi_distinct.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 @@ -187,7 +188,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out index c09764c156..177eefd4ea 100644 --- ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/cbo_rp_groupby3_noskew_multi_distinct.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/colstats_all_nulls.q.out ql/src/test/results/clientpositive/colstats_all_nulls.q.out index b56ad7fba9..77e4bf926d 100644 --- ql/src/test/results/clientpositive/colstats_all_nulls.q.out +++ ql/src/test/results/clientpositive/colstats_all_nulls.q.out @@ -30,10 +30,12 @@ POSTHOOK: Lineage: all_nulls.c SIMPLE [] PREHOOK: query: analyze table all_nulls compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@all_nulls +PREHOOK: Output: default@all_nulls #### A masked pattern was here #### POSTHOOK: query: analyze table all_nulls compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@all_nulls +POSTHOOK: Output: default@all_nulls #### A masked pattern was here #### PREHOOK: query: describe formatted all_nulls a PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out index 0d78ea2e70..ee0da96c12 100644 --- ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out +++ ql/src/test/results/clientpositive/columnStatsUpdateForStatsOptimizer_2.q.out @@ -55,10 +55,12 @@ Storage Desc Params: PREHOOK: query: analyze table calendar compute statistics for columns year PREHOOK: type: QUERY PREHOOK: Input: default@calendar +PREHOOK: Output: default@calendar #### A masked pattern was here #### POSTHOOK: query: analyze table calendar compute statistics for columns year POSTHOOK: type: QUERY POSTHOOK: Input: default@calendar +POSTHOOK: Output: default@calendar #### A masked pattern was here #### PREHOOK: query: desc formatted calendar PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out index 66ff79f2bd..a347b01981 100644 --- ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out +++ ql/src/test/results/clientpositive/column_pruner_multiple_children.q.out @@ -27,8 +27,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 - Stage-4 depends on stages: Stage-2, Stage-3 + Stage-2 depends on stages: Stage-0, Stage-3 Stage-3 depends on stages: Stage-1 STAGE PLANS: @@ -89,10 +88,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-4 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string diff --git ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out index 018e18fa9b..feac97e8eb 100644 --- ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out +++ ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out @@ -60,7 +60,8 @@ STAGE PLANS: name: default.columnarserde_create_shortcut Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE columnarserde_create_shortcut SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/columnstats_infinity.q.out ql/src/test/results/clientpositive/columnstats_infinity.q.out index 777345cb03..6b31882b4d 100644 --- ql/src/test/results/clientpositive/columnstats_infinity.q.out +++ ql/src/test/results/clientpositive/columnstats_infinity.q.out @@ -117,10 +117,12 @@ Storage Desc Params: PREHOOK: query: analyze table table_change_numeric_group_string_group_floating_string_group compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@table_change_numeric_group_string_group_floating_string_group +PREHOOK: Output: default@table_change_numeric_group_string_group_floating_string_group #### A masked pattern was here #### POSTHOOK: query: analyze table table_change_numeric_group_string_group_floating_string_group compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@table_change_numeric_group_string_group_floating_string_group +POSTHOOK: Output: default@table_change_numeric_group_string_group_floating_string_group #### A masked pattern was here #### PREHOOK: query: desc formatted table_change_numeric_group_string_group_floating_string_group PREHOOK: type: DESCTABLE @@ -273,7 +275,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"c1\":\"true\",\"c10\":\"true\",\"c12\":\"true\",\"c13\":\"true\",\"c15\":\"true\",\"c3\":\"true\",\"c4\":\"true\",\"c6\":\"true\",\"c7\":\"true\",\"c9\":\"true\",\"insert_num\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"c1\":\"true\",\"c10\":\"true\",\"c11\":\"true\",\"c12\":\"true\",\"c13\":\"true\",\"c14\":\"true\",\"c15\":\"true\",\"c2\":\"true\",\"c3\":\"true\",\"c4\":\"true\",\"c5\":\"true\",\"c6\":\"true\",\"c7\":\"true\",\"c8\":\"true\",\"c9\":\"true\",\"insert_num\":\"true\"}} numFiles 1 numRows 5 rawDataSize 1250 diff --git ql/src/test/results/clientpositive/columnstats_partlvl.q.out ql/src/test/results/clientpositive/columnstats_partlvl.q.out index 08f170913e..f15b44b758 100644 --- ql/src/test/results/clientpositive/columnstats_partlvl.q.out +++ ql/src/test/results/clientpositive/columnstats_partlvl.q.out @@ -83,7 +83,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -106,7 +107,8 @@ STAGE PLANS: TableScan alias: employee_part Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.employee_part/ + GatherStats: true Select Operator expressions: employeeid (type: int) outputColumnNames: employeeid @@ -146,10 +148,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.employee_part numFiles 1 - numRows 0 partition_columns employeesalary partition_columns.types double - rawDataSize 0 serialization.ddl struct employee_part { i32 employeeid, string employeename} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -215,7 +215,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.employee_part/ Column Stats Desc: Columns: employeeID Column Types: int @@ -226,12 +228,54 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) co PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### +PREHOOK: query: describe formatted Employee_Part partition(employeeSalary=2000.0) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@employee_part +POSTHOOK: query: describe formatted Employee_Part partition(employeeSalary=2000.0) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@employee_part +# col_name data_type comment +employeeid int +employeename string + +# Partition Information +# col_name data_type comment +employeesalary double + +# Detailed Partition Information +Partition Value: [2000.0] +Database: default +Table: employee_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"employeeid\":\"true\"}} + numFiles 1 + numRows 13 + rawDataSize 92 + totalSize 105 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim | + serialization.format | PREHOOK: query: explain analyze table Employee_Part partition (employeeSalary=4000.0) compute statistics for columns employeeID PREHOOK: type: QUERY @@ -285,7 +329,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -308,7 +353,8 @@ STAGE PLANS: TableScan alias: employee_part Statistics: Num rows: 26 Data size: 105 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.employee_part/ + GatherStats: true Select Operator expressions: employeeid (type: int) outputColumnNames: employeeid @@ -348,10 +394,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.employee_part numFiles 1 - numRows 0 partition_columns employeesalary partition_columns.types double - rawDataSize 0 serialization.ddl struct employee_part { i32 employeeid, string employeename} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -417,7 +461,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.employee_part/ Column Stats Desc: Columns: employeeID Column Types: int @@ -428,11 +474,15 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary=4000.0) co PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary=4000.0) compute statistics for columns employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: explain analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns @@ -450,22 +500,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string) outputColumnNames: employeeid, employeename - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') keys: 2000.0 (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: 2000.0 (type: double) sort order: + Map-reduce partition columns: 2000.0 (type: double) - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct), _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -473,21 +523,22 @@ STAGE PLANS: keys: 2000.0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: struct), _col2 (type: struct), 2000.0 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -497,11 +548,15 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) co PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary=2000.0) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary=2000.0) employeeID PREHOOK: type: DESCTABLE @@ -555,22 +610,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string), employeesalary (type: double) outputColumnNames: employeeid, employeename, employeesalary - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') keys: employeesalary (type: double) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double) sort order: + Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: struct), _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -578,21 +633,22 @@ STAGE PLANS: keys: KEY._col0 (type: double) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 105 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 92 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -603,12 +659,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition(employeeSalary=2000.0) employeeID PREHOOK: type: DESCTABLE @@ -662,11 +724,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string) outputColumnNames: employeeid, employeename - Statistics: Num rows: 2 Data size: 210 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 184 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') mode: hash @@ -691,7 +753,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -702,12 +765,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part employeeID PREHOOK: type: DESCTABLE @@ -744,11 +813,15 @@ PREHOOK: query: analyze table default.Employee_Part partition (employeeSalary=20 PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table default.Employee_Part partition (employeeSalary=2000.0) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 #### A masked pattern was here #### PREHOOK: query: describe formatted default.Employee_Part partition (employeeSalary=2000.0) employeeID PREHOOK: type: DESCTABLE @@ -774,12 +847,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0 PREHOOK: Input: default@employee_part@employeesalary=4000.0 +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0 +PREHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### POSTHOOK: query: analyze table default.Employee_Part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0 POSTHOOK: Input: default@employee_part@employeesalary=4000.0 +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0 +POSTHOOK: Output: default@employee_part@employeesalary=4000.0 #### A masked pattern was here #### PREHOOK: query: use default PREHOOK: type: SWITCHDATABASE diff --git ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out index a626597b29..414b715b7a 100644 --- ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out +++ ql/src/test/results/clientpositive/columnstats_partlvl_dp.q.out @@ -119,7 +119,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeName, employeeID Column Types: string, int @@ -129,12 +130,55 @@ PREHOOK: query: analyze table Employee_Part partition (employeeSalary='4000.0', PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary='4000.0', country) compute statistics for columns employeeName, employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### +PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='4000.0', country='USA') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@employee_part +POSTHOOK: query: describe formatted Employee_Part partition (employeeSalary='4000.0', country='USA') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@employee_part +# col_name data_type comment +employeeid int +employeename string + +# Partition Information +# col_name data_type comment +employeesalary double +country string + +# Detailed Partition Information +Partition Value: [4000.0, USA] +Database: default +Table: employee_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"employeeid\":\"true\",\"employeename\":\"true\"}} + numFiles 1 + numRows 7 + rawDataSize 57 + totalSize 64 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim | + serialization.format | PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='4000.0', country='USA') employeeName PREHOOK: type: DESCTABLE PREHOOK: Input: default@employee_part @@ -206,7 +250,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -217,12 +262,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@employee_part PREHOOK: Input: default@employee_part@employeesalary=2000.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary='2000.0') compute statistics for columns employeeID POSTHOOK: type: QUERY POSTHOOK: Input: default@employee_part POSTHOOK: Input: default@employee_part@employeesalary=2000.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='2000.0', country='USA') employeeID PREHOOK: type: DESCTABLE @@ -276,22 +327,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: PARTIAL Column stats: NONE Select Operator expressions: employeeid (type: int), employeesalary (type: double), country (type: string) outputColumnNames: employeeid, employeesalary, country - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll') keys: employeesalary (type: double), country (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 116 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -299,21 +350,22 @@ STAGE PLANS: keys: KEY._col0 (type: double), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: struct), _col0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 58 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeID Column Types: int @@ -328,6 +380,13 @@ PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA PREHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary) compute statistics for columns employeeID POSTHOOK: type: QUERY @@ -338,6 +397,13 @@ POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA POSTHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='3000.0', country='UK') employeeID PREHOOK: type: DESCTABLE @@ -373,22 +439,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: employee_part - Statistics: Num rows: 2 Data size: 466 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: employeeid (type: int), employeename (type: string), employeesalary (type: double), country (type: string) outputColumnNames: employeeid, employeename, employeesalary, country - Statistics: Num rows: 2 Data size: 466 Basic stats: PARTIAL Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(employeeid, 'hll'), compute_stats(employeename, 'hll') keys: employeesalary (type: double), country (type: string) mode: hash outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: double), _col1 (type: string) sort order: ++ Map-reduce partition columns: _col0 (type: double), _col1 (type: string) - Statistics: Num rows: 2 Data size: 466 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 412 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct), _col3 (type: struct) Reduce Operator Tree: Group By Operator @@ -396,21 +462,22 @@ STAGE PLANS: keys: KEY._col0 (type: double), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 233 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 206 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: employeeid, employeename Column Types: int, string @@ -425,6 +492,13 @@ PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA PREHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK PREHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee_part +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +PREHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee_Part partition (employeeSalary,country) compute statistics for columns POSTHOOK: type: QUERY @@ -435,6 +509,13 @@ POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=3000.0/country=USA POSTHOOK: Input: default@employee_part@employeesalary=3500.0/country=UK POSTHOOK: Input: default@employee_part@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee_part +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=3000.0/country=USA +POSTHOOK: Output: default@employee_part@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee_part@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee_Part partition (employeeSalary='3500.0', country='UK') employeeName PREHOOK: type: DESCTABLE @@ -511,6 +592,11 @@ PREHOOK: Input: default@employee@employeesalary=2000.0/country=UK PREHOOK: Input: default@employee@employeesalary=2000.0/country=USA PREHOOK: Input: default@employee@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee +PREHOOK: Output: default@employee@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=3500.0/country=UK #### A masked pattern was here #### POSTHOOK: query: analyze table Employee partition (employeeSalary,country) compute statistics for columns POSTHOOK: type: QUERY @@ -519,6 +605,11 @@ POSTHOOK: Input: default@employee@employeesalary=2000.0/country=UK POSTHOOK: Input: default@employee@employeesalary=2000.0/country=USA POSTHOOK: Input: default@employee@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=3500.0/country=UK #### A masked pattern was here #### PREHOOK: query: describe formatted Employee partition (employeeSalary='3500.0', country='UK') employeeName PREHOOK: type: DESCTABLE @@ -565,6 +656,13 @@ PREHOOK: Input: default@employee@employeesalary=3000.0/country=UK PREHOOK: Input: default@employee@employeesalary=3000.0/country=USA PREHOOK: Input: default@employee@employeesalary=3500.0/country=UK PREHOOK: Input: default@employee@employeesalary=4000.0/country=USA +PREHOOK: Output: default@employee +PREHOOK: Output: default@employee@employeesalary=2000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=2000.0/country=USA +PREHOOK: Output: default@employee@employeesalary=3000.0/country=UK +PREHOOK: Output: default@employee@employeesalary=3000.0/country=USA +PREHOOK: Output: default@employee@employeesalary=3500.0/country=UK +PREHOOK: Output: default@employee@employeesalary=4000.0/country=USA #### A masked pattern was here #### POSTHOOK: query: analyze table Employee partition (employeeSalary) compute statistics for columns POSTHOOK: type: QUERY @@ -575,6 +673,13 @@ POSTHOOK: Input: default@employee@employeesalary=3000.0/country=UK POSTHOOK: Input: default@employee@employeesalary=3000.0/country=USA POSTHOOK: Input: default@employee@employeesalary=3500.0/country=UK POSTHOOK: Input: default@employee@employeesalary=4000.0/country=USA +POSTHOOK: Output: default@employee +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=2000.0/country=USA +POSTHOOK: Output: default@employee@employeesalary=3000.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=3000.0/country=USA +POSTHOOK: Output: default@employee@employeesalary=3500.0/country=UK +POSTHOOK: Output: default@employee@employeesalary=4000.0/country=USA #### A masked pattern was here #### PREHOOK: query: describe formatted Employee partition (employeeSalary='3000.0', country='USA') employeeName PREHOOK: type: DESCTABLE @@ -615,11 +720,15 @@ PREHOOK: query: analyze table Employee partition (employeeSalary='6000.0',countr PREHOOK: type: QUERY PREHOOK: Input: default@employee PREHOOK: Input: default@employee@employeesalary=6000.0/country=UK +PREHOOK: Output: default@employee +PREHOOK: Output: default@employee@employeesalary=6000.0/country=UK #### A masked pattern was here #### POSTHOOK: query: analyze table Employee partition (employeeSalary='6000.0',country='UK') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@employee POSTHOOK: Input: default@employee@employeesalary=6000.0/country=UK +POSTHOOK: Output: default@employee +POSTHOOK: Output: default@employee@employeesalary=6000.0/country=UK #### A masked pattern was here #### PREHOOK: query: describe formatted Employee partition (employeeSalary='6000.0', country='UK') employeeName PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/columnstats_quoting.q.out ql/src/test/results/clientpositive/columnstats_quoting.q.out index b17ce0e67a..683c1e274f 100644 --- ql/src/test/results/clientpositive/columnstats_quoting.q.out +++ ql/src/test/results/clientpositive/columnstats_quoting.q.out @@ -53,7 +53,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: user id, user name Column Types: bigint, string @@ -62,10 +63,12 @@ STAGE PLANS: PREHOOK: query: analyze table user_web_events compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@user_web_events +PREHOOK: Output: default@user_web_events #### A masked pattern was here #### POSTHOOK: query: analyze table user_web_events compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@user_web_events +POSTHOOK: Output: default@user_web_events #### A masked pattern was here #### PREHOOK: query: explain analyze table user_web_events compute statistics for columns `user id` PREHOOK: type: QUERY @@ -110,7 +113,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: user id Column Types: bigint @@ -119,8 +123,10 @@ STAGE PLANS: PREHOOK: query: analyze table user_web_events compute statistics for columns `user id` PREHOOK: type: QUERY PREHOOK: Input: default@user_web_events +PREHOOK: Output: default@user_web_events #### A masked pattern was here #### POSTHOOK: query: analyze table user_web_events compute statistics for columns `user id` POSTHOOK: type: QUERY POSTHOOK: Input: default@user_web_events +POSTHOOK: Output: default@user_web_events #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/columnstats_tbllvl.q.out ql/src/test/results/clientpositive/columnstats_tbllvl.q.out index f51ff4cbea..3f9776f97c 100644 --- ql/src/test/results/clientpositive/columnstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/columnstats_tbllvl.q.out @@ -54,11 +54,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_web_text_none - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(avgtimeonsite, 'hll'), compute_stats(adrevenue, 'hll') mode: hash @@ -83,7 +83,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -105,12 +106,13 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_web_text_none - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics Aggregation Key Prefix: default.uservisits_web_text_none/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(avgtimeonsite, 'hll'), compute_stats(adrevenue, 'hll') mode: hash @@ -141,8 +143,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.uservisits_web_text_none numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct uservisits_web_text_none { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -162,8 +162,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.uservisits_web_text_none numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct uservisits_web_text_none { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -205,7 +203,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.uservisits_web_text_none/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -215,10 +215,12 @@ STAGE PLANS: PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: explain analyze table default.UserVisits_web_text_none compute statistics for columns @@ -236,11 +238,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_web_text_none - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), desturl (type: string), visitdate (type: string), adrevenue (type: float), useragent (type: string), ccode (type: string), lcode (type: string), skeyword (type: string), avgtimeonsite (type: int) outputColumnNames: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(desturl, 'hll'), compute_stats(visitdate, 'hll'), compute_stats(adrevenue, 'hll'), compute_stats(useragent, 'hll'), compute_stats(ccode, 'hll'), compute_stats(lcode, 'hll'), compute_stats(skeyword, 'hll'), compute_stats(avgtimeonsite, 'hll') mode: hash @@ -265,7 +267,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite Column Types: string, string, string, float, string, string, string, string, int @@ -274,10 +277,12 @@ STAGE PLANS: PREHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: describe formatted UserVisits_web_text_none destURL PREHOOK: type: DESCTABLE @@ -297,7 +302,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted UserVisits_web_text_none adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -316,7 +321,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted UserVisits_web_text_none avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -335,7 +340,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: CREATE TABLE empty_tab( a int, b double, @@ -401,7 +406,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d, e Column Types: int, double, string, boolean, binary @@ -410,10 +416,12 @@ STAGE PLANS: PREHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e PREHOOK: type: QUERY PREHOOK: Input: default@empty_tab +PREHOOK: Output: default@empty_tab #### A masked pattern was here #### POSTHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e POSTHOOK: type: QUERY POSTHOOK: Input: default@empty_tab +POSTHOOK: Output: default@empty_tab #### A masked pattern was here #### PREHOOK: query: create database if not exists dummydb PREHOOK: type: CREATEDATABASE @@ -430,10 +438,12 @@ POSTHOOK: Input: database:dummydb PREHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns destURL PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table default.UserVisits_web_text_none compute statistics for columns destURL POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: describe formatted default.UserVisits_web_text_none destURL PREHOOK: type: DESCTABLE @@ -453,7 +463,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: CREATE TABLE UserVisits_in_dummy_db ( sourceIP string, destURL string, @@ -512,11 +522,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_in_dummy_db - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(avgtimeonsite, 'hll'), compute_stats(adrevenue, 'hll') mode: hash @@ -541,7 +551,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -563,12 +574,13 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_in_dummy_db - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics Aggregation Key Prefix: dummydb.uservisits_in_dummy_db/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(avgtimeonsite, 'hll'), compute_stats(adrevenue, 'hll') mode: hash @@ -599,8 +611,6 @@ STAGE PLANS: #### A masked pattern was here #### name dummydb.uservisits_in_dummy_db numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct uservisits_in_dummy_db { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -620,8 +630,6 @@ STAGE PLANS: #### A masked pattern was here #### name dummydb.uservisits_in_dummy_db numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct uservisits_in_dummy_db { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -663,7 +671,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: dummydb.uservisits_in_dummy_db/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -673,10 +683,12 @@ STAGE PLANS: PREHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: dummydb@uservisits_in_dummy_db +PREHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### POSTHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: dummydb@uservisits_in_dummy_db +POSTHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### PREHOOK: query: explain analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns @@ -694,11 +706,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_in_dummy_db - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), desturl (type: string), visitdate (type: string), adrevenue (type: float), useragent (type: string), ccode (type: string), lcode (type: string), skeyword (type: string), avgtimeonsite (type: int) outputColumnNames: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 7005 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(desturl, 'hll'), compute_stats(visitdate, 'hll'), compute_stats(adrevenue, 'hll'), compute_stats(useragent, 'hll'), compute_stats(ccode, 'hll'), compute_stats(lcode, 'hll'), compute_stats(skeyword, 'hll'), compute_stats(avgtimeonsite, 'hll') mode: hash @@ -723,7 +735,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceip, desturl, visitdate, adrevenue, useragent, ccode, lcode, skeyword, avgtimeonsite Column Types: string, string, string, float, string, string, string, string, int @@ -732,10 +745,12 @@ STAGE PLANS: PREHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: dummydb@uservisits_in_dummy_db +PREHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### POSTHOOK: query: analyze table dummydb.UserVisits_in_dummy_db compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: dummydb@uservisits_in_dummy_db +POSTHOOK: Output: dummydb@uservisits_in_dummy_db #### A masked pattern was here #### PREHOOK: query: describe formatted dummydb.UserVisits_in_dummy_db destURL PREHOOK: type: DESCTABLE @@ -755,7 +770,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted dummydb.UserVisits_in_dummy_db adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: dummydb@uservisits_in_dummy_db @@ -774,7 +789,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: describe formatted dummydb.UserVisits_in_dummy_db avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: dummydb@uservisits_in_dummy_db @@ -793,7 +808,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} PREHOOK: query: drop table dummydb.UserVisits_in_dummy_db PREHOOK: type: DROPTABLE PREHOOK: Input: dummydb@uservisits_in_dummy_db diff --git ql/src/test/results/clientpositive/compustat_avro.q.out ql/src/test/results/clientpositive/compustat_avro.q.out index cdf9d36672..f168cc5d50 100644 --- ql/src/test/results/clientpositive/compustat_avro.q.out +++ ql/src/test/results/clientpositive/compustat_avro.q.out @@ -46,10 +46,12 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"col1\":\"tru PREHOOK: query: analyze table testAvro compute statistics for columns col1,col3 PREHOOK: type: QUERY PREHOOK: Input: default@testavro +PREHOOK: Output: default@testavro #### A masked pattern was here #### POSTHOOK: query: analyze table testAvro compute statistics for columns col1,col3 POSTHOOK: type: QUERY POSTHOOK: Input: default@testavro +POSTHOOK: Output: default@testavro #### A masked pattern was here #### PREHOOK: query: describe formatted testAvro col1 PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/compute_stats_date.q.out ql/src/test/results/clientpositive/compute_stats_date.q.out index 1e6de6198a..796b3d3e5f 100644 --- ql/src/test/results/clientpositive/compute_stats_date.q.out +++ ql/src/test/results/clientpositive/compute_stats_date.q.out @@ -89,7 +89,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: fl_date Column Types: date @@ -98,10 +99,12 @@ STAGE PLANS: PREHOOK: query: analyze table tab_date compute statistics for columns fl_date PREHOOK: type: QUERY PREHOOK: Input: default@tab_date +PREHOOK: Output: default@tab_date #### A masked pattern was here #### POSTHOOK: query: analyze table tab_date compute statistics for columns fl_date POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_date +POSTHOOK: Output: default@tab_date #### A masked pattern was here #### PREHOOK: query: describe formatted tab_date fl_date PREHOOK: type: DESCTABLE @@ -121,7 +124,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"fl_date\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"fl_date\":\"true\"}} PREHOOK: query: alter table tab_date update statistics for column fl_date set ('numDVs'='19', 'highValue'='2015-01-01', 'lowValue'='0') PREHOOK: type: ALTERTABLE_UPDATETABLESTATS POSTHOOK: query: alter table tab_date update statistics for column fl_date set ('numDVs'='19', 'highValue'='2015-01-01', 'lowValue'='0') @@ -144,4 +147,4 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"fl_date\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"fl_date\":\"true\"}} diff --git ql/src/test/results/clientpositive/constGby.q.out ql/src/test/results/clientpositive/constGby.q.out index 7115be3611..c633624935 100644 --- ql/src/test/results/clientpositive/constGby.q.out +++ ql/src/test/results/clientpositive/constGby.q.out @@ -17,10 +17,12 @@ POSTHOOK: Output: default@t1 PREHOOK: query: analyze table t1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 #### A masked pattern was here #### POSTHOOK: query: analyze table t1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 #### A masked pattern was here #### PREHOOK: query: explain select count(1) from t1 group by 1 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/constant_prop_2.q.out ql/src/test/results/clientpositive/constant_prop_2.q.out index 77a91222fb..7c452451bd 100644 --- ql/src/test/results/clientpositive/constant_prop_2.q.out +++ ql/src/test/results/clientpositive/constant_prop_2.q.out @@ -74,7 +74,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string diff --git ql/src/test/results/clientpositive/constant_prop_3.q.out ql/src/test/results/clientpositive/constant_prop_3.q.out index 4b3de33e91..cba4744866 100644 --- ql/src/test/results/clientpositive/constant_prop_3.q.out +++ ql/src/test/results/clientpositive/constant_prop_3.q.out @@ -51,10 +51,12 @@ POSTHOOK: Output: default@part_hive PREHOOK: query: analyze table part_hive compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@part_hive +PREHOOK: Output: default@part_hive #### A masked pattern was here #### POSTHOOK: query: analyze table part_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@part_hive +POSTHOOK: Output: default@part_hive #### A masked pattern was here #### PREHOOK: query: analyze table partsupp_hive compute statistics PREHOOK: type: QUERY @@ -67,10 +69,12 @@ POSTHOOK: Output: default@partsupp_hive PREHOOK: query: analyze table partsupp_hive compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@partsupp_hive +PREHOOK: Output: default@partsupp_hive #### A masked pattern was here #### POSTHOOK: query: analyze table partsupp_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partsupp_hive +POSTHOOK: Output: default@partsupp_hive #### A masked pattern was here #### PREHOOK: query: analyze table supplier_hive compute statistics PREHOOK: type: QUERY @@ -83,10 +87,12 @@ POSTHOOK: Output: default@supplier_hive PREHOOK: query: analyze table supplier_hive compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@supplier_hive +PREHOOK: Output: default@supplier_hive #### A masked pattern was here #### POSTHOOK: query: analyze table supplier_hive compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@supplier_hive +POSTHOOK: Output: default@supplier_hive #### A masked pattern was here #### Warning: Shuffle Join JOIN[25][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: explain select diff --git ql/src/test/results/clientpositive/constprog_dp.q.out ql/src/test/results/clientpositive/constprog_dp.q.out index 8cf301d6f7..86b6c326d2 100644 --- ql/src/test/results/clientpositive/constprog_dp.q.out +++ ql/src/test/results/clientpositive/constprog_dp.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/constprog_type.q.out ql/src/test/results/clientpositive/constprog_type.q.out index d145d37a3b..27ef1f482b 100644 --- ql/src/test/results/clientpositive/constprog_type.q.out +++ ql/src/test/results/clientpositive/constprog_type.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/correlated_join_keys.q.out ql/src/test/results/clientpositive/correlated_join_keys.q.out index b81a5611b1..aaea8456d9 100644 --- ql/src/test/results/clientpositive/correlated_join_keys.q.out +++ ql/src/test/results/clientpositive/correlated_join_keys.q.out @@ -61,10 +61,12 @@ POSTHOOK: Output: default@customer_address PREHOOK: query: analyze table customer_address compute statistics for columns ca_state, ca_zip PREHOOK: type: QUERY PREHOOK: Input: default@customer_address +PREHOOK: Output: default@customer_address #### A masked pattern was here #### POSTHOOK: query: analyze table customer_address compute statistics for columns ca_state, ca_zip POSTHOOK: type: QUERY POSTHOOK: Input: default@customer_address +POSTHOOK: Output: default@customer_address #### A masked pattern was here #### PREHOOK: query: explain select count(*) from customer_address a join customer_address b on (a.ca_zip = b.ca_zip and a.ca_state = b.ca_state) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/correlationoptimizer5.q.out ql/src/test/results/clientpositive/correlationoptimizer5.q.out index 8c2e6cf4d3..9037b106ab 100644 --- ql/src/test/results/clientpositive/correlationoptimizer5.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer5.q.out @@ -116,36 +116,35 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: n - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE TableScan - alias: m - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + alias: y + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -153,18 +152,14 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + outputColumnNames: _col0 + Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -174,14 +169,14 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -189,15 +184,15 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col1 (type: string) + expressions: _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -215,41 +210,43 @@ STAGE PLANS: name: default.dest_co1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + alias: m + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE TableScan - alias: y - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + alias: n + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -257,14 +254,18 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + outputColumnNames: _col0, _col2 + Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe PREHOOK: query: INSERT OVERWRITE TABLE dest_co1 SELECT b.key, d.val @@ -322,79 +323,109 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: n - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE TableScan - alias: m - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + alias: y + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE TableScan - alias: x - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + alias: m + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE TableScan - alias: y - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + alias: n + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 4 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2956 Data size: 12099 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Mux Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_co2 Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col1 (type: string) + expressions: _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Mux Operator @@ -405,10 +436,10 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 + outputColumnNames: _col0, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col1 (type: string) + expressions: _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE File Output Operator @@ -419,36 +450,6 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest_co2 - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Mux Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 _col0 (type: int) - 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.dest_co2 Stage: Stage-0 Move Operator @@ -461,7 +462,8 @@ STAGE PLANS: name: default.dest_co2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_co2 SELECT b.key, d.val @@ -527,21 +529,21 @@ STAGE PLANS: Stage: Stage-14 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_0:$hdt$_1:m + $hdt$_0:$hdt$_1:y Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_0:$hdt$_1:m + $hdt$_0:$hdt$_1:y TableScan - alias: m - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + alias: y + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 216 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1447 Data size: 5791 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col0 (type: int) @@ -551,33 +553,29 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: n - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), val (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1453 Data size: 5812 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + outputColumnNames: _col0 + Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work @@ -608,15 +606,15 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col1 (type: string) + expressions: _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -636,7 +634,8 @@ STAGE PLANS: name: default.dest_co3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-13 Map Reduce Local Work @@ -662,15 +661,15 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col1 (type: string) + expressions: _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -687,14 +686,14 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 308 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 1598 Data size: 6393 Basic stats: COMPLETE Column stats: NONE TableScan Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 6393 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -702,15 +701,15 @@ STAGE PLANS: keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col1, _col2 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + outputColumnNames: _col0, _col2 + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col2 (type: int), _col1 (type: string) + expressions: _col0 (type: int), _col2 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 338 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1757 Data size: 7032 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -720,21 +719,21 @@ STAGE PLANS: Stage: Stage-15 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_1:$hdt$_2:y + $hdt$_1:$hdt$_1:m Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_1:$hdt$_2:y + $hdt$_1:$hdt$_1:m TableScan - alias: y - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + alias: m + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5791 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 54 Data size: 216 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col0 (type: int) @@ -744,29 +743,33 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + alias: n + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: int), val (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 280 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 keys: 0 _col0 (type: int) 1 _col0 (type: int) - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 6393 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + outputColumnNames: _col0, _col2 + Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col2 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 59 Data size: 237 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Local Work: Map Reduce Local Work diff --git ql/src/test/results/clientpositive/cp_sel.q.out ql/src/test/results/clientpositive/cp_sel.q.out index af2efeb0cf..910003778a 100644 --- ql/src/test/results/clientpositive/cp_sel.q.out +++ ql/src/test/results/clientpositive/cp_sel.q.out @@ -128,7 +128,8 @@ STAGE PLANS: name: default.testpartbucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table testpartbucket partition(ds,hr) select key,value,'hello' as ds, 'world' as hr from srcpart where hr=11 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/ctas.q.out ql/src/test/results/clientpositive/ctas.q.out index 1764c6b5e6..e65965019d 100644 --- ql/src/test/results/clientpositive/ctas.q.out +++ ql/src/test/results/clientpositive/ctas.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.nzhang_CTAS1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -248,7 +249,8 @@ STAGE PLANS: name: default.nzhang_ctas2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas2 as select * from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -398,7 +400,8 @@ STAGE PLANS: name: default.nzhang_ctas3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -612,7 +615,8 @@ STAGE PLANS: name: default.nzhang_ctas4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -766,7 +770,8 @@ STAGE PLANS: name: default.nzhang_ctas5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/ctas_colname.q.out ql/src/test/results/clientpositive/ctas_colname.q.out index 795517df12..144811d866 100644 --- ql/src/test/results/clientpositive/ctas_colname.q.out +++ ql/src/test/results/clientpositive/ctas_colname.q.out @@ -62,7 +62,8 @@ STAGE PLANS: name: default.summary Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT @@ -221,7 +222,8 @@ STAGE PLANS: name: default.x4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x4 as select *, rank() over(partition by key order by value) as rr from src1 PREHOOK: type: CREATETABLE_AS_SELECT @@ -411,7 +413,8 @@ STAGE PLANS: name: default.x5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x5 as select *, lead(key,1) over(partition by key order by value) as lead1 from src limit 20 PREHOOK: type: CREATETABLE_AS_SELECT @@ -551,7 +554,8 @@ STAGE PLANS: name: default.x6 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -731,7 +735,8 @@ STAGE PLANS: name: default.x7 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x7 as select * from (select *, count(value) from src group by key, value) a PREHOOK: type: CREATETABLE_AS_SELECT @@ -1164,7 +1169,8 @@ STAGE PLANS: name: default.x8 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x8 as select * from (select *, count(value) from src group by key, value having key < 9) a PREHOOK: type: CREATETABLE_AS_SELECT @@ -1300,7 +1306,8 @@ STAGE PLANS: name: default.x9 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table x9 as select * from (select max(value),key from src group by key having key < 9 AND max(value) IS NOT NULL) a PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/ctas_uses_database_location.q.out ql/src/test/results/clientpositive/ctas_uses_database_location.q.out index fe5cf87263..ad4402af76 100644 --- ql/src/test/results/clientpositive/ctas_uses_database_location.q.out +++ ql/src/test/results/clientpositive/ctas_uses_database_location.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: db1.table_db1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/decimal_stats.q.out ql/src/test/results/clientpositive/decimal_stats.q.out index 9d7e750e40..b84171b611 100644 --- ql/src/test/results/clientpositive/decimal_stats.q.out +++ ql/src/test/results/clientpositive/decimal_stats.q.out @@ -35,10 +35,12 @@ POSTHOOK: Lineage: decimal_1.v EXPRESSION [] PREHOOK: query: analyze table decimal_1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@decimal_1 +PREHOOK: Output: default@decimal_1 #### A masked pattern was here #### POSTHOOK: query: analyze table decimal_1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_1 +POSTHOOK: Output: default@decimal_1 #### A masked pattern was here #### PREHOOK: query: desc formatted decimal_1 v PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/deleteAnalyze.q.out ql/src/test/results/clientpositive/deleteAnalyze.q.out index 840f0533ef..1d051fd15e 100644 --- ql/src/test/results/clientpositive/deleteAnalyze.q.out +++ ql/src/test/results/clientpositive/deleteAnalyze.q.out @@ -87,18 +87,58 @@ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 +PREHOOK: Output: default@testdeci2 #### A masked pattern was here #### POSTHOOK: query: analyze table testdeci2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 +POSTHOOK: Output: default@testdeci2 +#### A masked pattern was here #### +PREHOOK: query: describe formatted testdeci2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testdeci2 +POSTHOOK: query: describe formatted testdeci2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testdeci2 +# col_name data_type comment +id int +amount decimal(10,3) +sales_tax decimal(10,3) +item string + +# Detailed Table Information +Database: default #### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"amount\":\"true\",\"id\":\"true\",\"item\":\"true\",\"sales_tax\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 +PREHOOK: Output: default@testdeci2 #### A masked pattern was here #### POSTHOOK: query: analyze table testdeci2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 +POSTHOOK: Output: default@testdeci2 #### A masked pattern was here #### PREHOOK: query: explain select s.id, @@ -124,23 +164,23 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: item is not null (type: boolean) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: id (type: int), item (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int) TableScan alias: d - Statistics: Num rows: 2 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((id = 2) and item is not null) (type: boolean) Statistics: Num rows: 1 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE @@ -162,14 +202,14 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col3 (type: string) outputColumnNames: _col0, _col3, _col4 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), COALESCE(_col3,0) (type: decimal(13,3)), COALESCE(_col4,0) (type: decimal(13,3)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 456 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out index 65af3f8e4c..254d17b31e 100644 --- ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/display_colstats_tbllvl.q.out @@ -79,11 +79,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_web_text_none - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(avgtimeonsite, 'hll'), compute_stats(adrevenue, 'hll') mode: hash @@ -108,7 +108,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -130,12 +131,13 @@ STAGE PLANS: Map Operator Tree: TableScan alias: uservisits_web_text_none - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics Aggregation Key Prefix: default.uservisits_web_text_none/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite - Statistics: Num rows: 1 Data size: 7060 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(sourceip, 'hll'), compute_stats(avgtimeonsite, 'hll'), compute_stats(adrevenue, 'hll') mode: hash @@ -166,8 +168,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.uservisits_web_text_none numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct uservisits_web_text_none { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -187,8 +187,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.uservisits_web_text_none numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct uservisits_web_text_none { string sourceip, string desturl, string visitdate, float adrevenue, string useragent, string ccode, string lcode, string skeyword, i32 avgtimeonsite} serialization.format | serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -230,7 +228,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.uservisits_web_text_none/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -240,10 +240,12 @@ STAGE PLANS: PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE @@ -263,7 +265,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -282,7 +284,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -301,7 +303,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: CREATE TABLE empty_tab( a int, b double, @@ -386,7 +388,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d, e Column Types: int, double, string, boolean, binary @@ -395,10 +398,12 @@ STAGE PLANS: PREHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e PREHOOK: type: QUERY PREHOOK: Input: default@empty_tab +PREHOOK: Output: default@empty_tab #### A masked pattern was here #### POSTHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e POSTHOOK: type: QUERY POSTHOOK: Input: default@empty_tab +POSTHOOK: Output: default@empty_tab #### A masked pattern was here #### PREHOOK: query: desc formatted empty_tab a PREHOOK: type: DESCTABLE @@ -507,7 +512,7 @@ POSTHOOK: query: desc extended default.UserVisits_web_text_none sourceIP POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@uservisits_web_text_none sourceIP string from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -562,15 +567,17 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"sourceip\":\"true\"}} PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword PREHOOK: type: QUERY PREHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +PREHOOK: Output: test@uservisits_web_text_none POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword POSTHOOK: type: QUERY POSTHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +POSTHOOK: Output: test@uservisits_web_text_none PREHOOK: query: desc extended UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -578,7 +585,7 @@ POSTHOOK: query: desc extended UserVisits_web_text_none sKeyword POSTHOOK: type: DESCTABLE POSTHOOK: Input: test@uservisits_web_text_none sKeyword string from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"skeyword\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"skeyword\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -597,7 +604,7 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"skeyword\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"skeyword\":\"true\"}} PREHOOK: query: desc formatted test.UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -616,4 +623,4 @@ num_trues num_falses bitVector HL comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"skeyword\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"skeyword\":\"true\"}} diff --git ql/src/test/results/clientpositive/distinct_stats.q.out ql/src/test/results/clientpositive/distinct_stats.q.out index 73b4addf57..bc2ab02ef5 100644 --- ql/src/test/results/clientpositive/distinct_stats.q.out +++ ql/src/test/results/clientpositive/distinct_stats.q.out @@ -19,10 +19,12 @@ POSTHOOK: Lineage: t1.b SIMPLE [(src)src.FieldSchema(name:value, type:string, co PREHOOK: query: analyze table t1 compute statistics for columns a,b PREHOOK: type: QUERY PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 #### A masked pattern was here #### POSTHOOK: query: analyze table t1 compute statistics for columns a,b POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 #### A masked pattern was here #### PREHOOK: query: explain select count(distinct b) from t1 group by a diff --git ql/src/test/results/clientpositive/drop_table_with_stats.q.out ql/src/test/results/clientpositive/drop_table_with_stats.q.out index 52aa10a070..119f74a87e 100644 --- ql/src/test/results/clientpositive/drop_table_with_stats.q.out +++ ql/src/test/results/clientpositive/drop_table_with_stats.q.out @@ -30,10 +30,12 @@ PREHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb1@testtable #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb1@testtable POSTHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb1@testtable #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb1@testtable PREHOOK: query: CREATE TABLE IF NOT EXISTS TestTable1 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:tblstatsdb1 @@ -54,10 +56,12 @@ PREHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb1@testtable1 POSTHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb1@testtable1 PREHOOK: query: CREATE TABLE IF NOT EXISTS TESTTABLE2 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:tblstatsdb1 @@ -78,10 +82,12 @@ PREHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb1@testtable2 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb1@testtable2 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb1@testtable2 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb1@testtable2 PREHOOK: query: DROP TABLE tblstatsdb1.testtable PREHOOK: type: DROPTABLE PREHOOK: Input: tblstatsdb1@testtable @@ -146,10 +152,12 @@ PREHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb2@testtable #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb2@testtable POSTHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb2@testtable #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb2@testtable PREHOOK: query: CREATE TABLE IF NOT EXISTS TestTable1 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: TBLSTATSDB2@TestTable1 @@ -170,10 +178,12 @@ PREHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb2@testtable1 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb2@testtable1 POSTHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb2@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb2@testtable1 PREHOOK: query: CREATE TABLE IF NOT EXISTS TESTTABLE2 (key STRING, value STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: TBLSTATSDB2@TESTTABLE2 @@ -194,10 +204,12 @@ PREHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: tblstatsdb2@testtable2 #### A masked pattern was here #### +PREHOOK: Output: tblstatsdb2@testtable2 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: tblstatsdb2@testtable2 #### A masked pattern was here #### +POSTHOOK: Output: tblstatsdb2@testtable2 PREHOOK: query: DROP TABLE TBLSTATSDB2.testtable PREHOOK: type: DROPTABLE PREHOOK: Input: tblstatsdb2@testtable diff --git ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 76d0b7b02f..8199235519 100644 --- ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -69,5 +69,6 @@ STAGE PLANS: name: default.non_acid Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out index 9fefca131e..f1654d425b 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out @@ -546,22 +546,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) null sort order: a sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col0 (type: string), _col1 (type: string) auto parallelism: false @@ -641,6 +641,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 column.name.delimiter , columns key,value @@ -649,8 +650,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.src numFiles 1 - numRows 0 - rawDataSize 0 + numRows 500 + rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -661,6 +662,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 column.name.delimiter , columns key,value @@ -669,8 +671,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.src numFiles 1 - numRows 0 - rawDataSize 0 + numRows 500 + rawDataSize 5312 serialization.ddl struct src { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out index ebe50859b5..4eee575394 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out @@ -48,10 +48,12 @@ src PREHOOK: query: ANALYZE TABLE encrypted_table COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@encrypted_table +PREHOOK: Output: default@encrypted_table #### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging POSTHOOK: query: ANALYZE TABLE encrypted_table COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@encrypted_table +POSTHOOK: Output: default@encrypted_table #### A PARTIAL masked pattern was here #### data/warehouse/encrypted_table/.hive-staging PREHOOK: query: DESCRIBE FORMATTED encrypted_table key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out index 5fd06afe90..90d42fb468 100644 --- ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out +++ ql/src/test/results/clientpositive/exec_parallel_column_stats.q.out @@ -1,6 +1,18 @@ -PREHOOK: query: explain analyze table src compute statistics for columns +PREHOOK: query: create table t as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: t.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain analyze table t compute statistics for columns PREHOOK: type: QUERY -POSTHOOK: query: explain analyze table src compute statistics for columns +POSTHOOK: query: explain analyze table t compute statistics for columns POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage @@ -11,7 +23,7 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: src + alias: t Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) @@ -41,17 +53,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string - Table: default.src + Table: default.t -PREHOOK: query: analyze table src compute statistics for columns +PREHOOK: query: analyze table t compute statistics for columns PREHOOK: type: QUERY -PREHOOK: Input: default@src +PREHOOK: Input: default@t +PREHOOK: Output: default@t #### A masked pattern was here #### -POSTHOOK: query: analyze table src compute statistics for columns +POSTHOOK: query: analyze table t compute statistics for columns POSTHOOK: type: QUERY -POSTHOOK: Input: default@src +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/explain_ddl.q.out ql/src/test/results/clientpositive/explain_ddl.q.out index e108e2207c..f0e54c5153 100644 --- ql/src/test/results/clientpositive/explain_ddl.q.out +++ ql/src/test/results/clientpositive/explain_ddl.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.M1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -194,7 +195,8 @@ STAGE PLANS: name: default.M1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -288,7 +290,8 @@ STAGE PLANS: name: default.M1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -380,7 +383,8 @@ STAGE PLANS: name: default.V1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -540,7 +544,8 @@ STAGE PLANS: name: default.m1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out index b12d3a1f20..5f427375da 100644 --- ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out +++ ql/src/test/results/clientpositive/extrapolate_part_stats_date.q.out @@ -33,6 +33,10 @@ PREHOOK: Input: default@date_dim PREHOOK: Input: default@date_dim@d_date_sk=2416945 PREHOOK: Input: default@date_dim@d_date_sk=2416946 PREHOOK: Input: default@date_dim@d_date_sk=2416947 +PREHOOK: Output: default@date_dim +PREHOOK: Output: default@date_dim@d_date_sk=2416945 +PREHOOK: Output: default@date_dim@d_date_sk=2416946 +PREHOOK: Output: default@date_dim@d_date_sk=2416947 #### A masked pattern was here #### POSTHOOK: query: analyze table date_dim partition(d_date_sk) compute statistics for columns POSTHOOK: type: QUERY @@ -40,6 +44,10 @@ POSTHOOK: Input: default@date_dim POSTHOOK: Input: default@date_dim@d_date_sk=2416945 POSTHOOK: Input: default@date_dim@d_date_sk=2416946 POSTHOOK: Input: default@date_dim@d_date_sk=2416947 +POSTHOOK: Output: default@date_dim +POSTHOOK: Output: default@date_dim@d_date_sk=2416945 +POSTHOOK: Output: default@date_dim@d_date_sk=2416946 +POSTHOOK: Output: default@date_dim@d_date_sk=2416947 #### A masked pattern was here #### PREHOOK: query: explain select count(*) from date_dim where d_date > date "1900-01-02" and d_date_sk= 2416945 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out index 8933a7da1b..25f5372426 100644 --- ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out +++ ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out @@ -59,21 +59,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2001') state PREHOOK: type: DESCTABLE @@ -356,41 +364,57 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### PREHOOK: query: explain extended select state from loc_orc_2d PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out index 8427b2d673..5f74166e89 100644 --- ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out +++ ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out @@ -67,21 +67,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2002') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2002 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2002 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2002') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2002 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2002 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2001') state PREHOOK: type: DESCTABLE @@ -529,21 +537,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2003') compute statistics for columns state PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2003 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2003 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2003') compute statistics for columns state POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2003 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2003 #### A masked pattern was here #### PREHOOK: query: explain extended select state from loc_orc_1d PREHOOK: type: QUERY @@ -1009,21 +1025,29 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2002') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2002 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2002 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2002') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2002 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2002 #### A masked pattern was here #### PREHOOK: query: explain extended select state from loc_orc_2d PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/fm-sketch.q.out ql/src/test/results/clientpositive/fm-sketch.q.out index e519084ac6..7691342cac 100644 --- ql/src/test/results/clientpositive/fm-sketch.q.out +++ ql/src/test/results/clientpositive/fm-sketch.q.out @@ -58,7 +58,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -67,10 +68,12 @@ STAGE PLANS: PREHOOK: query: analyze table n compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@n +PREHOOK: Output: default@n #### A masked pattern was here #### POSTHOOK: query: analyze table n compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@n +POSTHOOK: Output: default@n #### A masked pattern was here #### PREHOOK: query: desc formatted n key PREHOOK: type: DESCTABLE @@ -151,7 +154,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -160,10 +164,12 @@ STAGE PLANS: PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -212,10 +218,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -264,10 +272,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -342,10 +352,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(values__tmp__table__5)values__tmp__table__ PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/groupby1.q.out ql/src/test/results/clientpositive/groupby1.q.out index 46e09dd256..d05feec784 100644 --- ql/src/test/results/clientpositive/groupby1.q.out +++ ql/src/test/results/clientpositive/groupby1.q.out @@ -90,7 +90,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby10.q.out ql/src/test/results/clientpositive/groupby10.q.out index d48b7db821..9f90a2ca3c 100644 --- ql/src/test/results/clientpositive/groupby10.q.out +++ ql/src/test/results/clientpositive/groupby10.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -197,7 +198,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -376,7 +378,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -442,7 +445,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -604,7 +608,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -617,7 +622,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, sum(distinct substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key diff --git ql/src/test/results/clientpositive/groupby11.q.out ql/src/test/results/clientpositive/groupby11.q.out index 1d0e86ab7d..2ab4c39c89 100644 --- ql/src/test/results/clientpositive/groupby11.q.out +++ ql/src/test/results/clientpositive/groupby11.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -189,7 +190,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 partition(ds='111') diff --git ql/src/test/results/clientpositive/groupby12.q.out ql/src/test/results/clientpositive/groupby12.q.out index 921fc92b3c..2f633f01a7 100644 --- ql/src/test/results/clientpositive/groupby12.q.out +++ ql/src/test/results/clientpositive/groupby12.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT COUNT(src.key), COUNT(DISTINCT value) GROUP BY src.key diff --git ql/src/test/results/clientpositive/groupby1_limit.q.out ql/src/test/results/clientpositive/groupby1_limit.q.out index 78a49ebf6c..e5fa6d1451 100644 --- ql/src/test/results/clientpositive/groupby1_limit.q.out +++ ql/src/test/results/clientpositive/groupby1_limit.q.out @@ -100,7 +100,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key ORDER BY src.key LIMIT 5 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_map.q.out ql/src/test/results/clientpositive/groupby1_map.q.out index cc985a5def..337c2e0fe1 100644 --- ql/src/test/results/clientpositive/groupby1_map.q.out +++ ql/src/test/results/clientpositive/groupby1_map.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_map_nomap.q.out ql/src/test/results/clientpositive/groupby1_map_nomap.q.out index cc985a5def..337c2e0fe1 100644 --- ql/src/test/results/clientpositive/groupby1_map_nomap.q.out +++ ql/src/test/results/clientpositive/groupby1_map_nomap.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_map_skew.q.out ql/src/test/results/clientpositive/groupby1_map_skew.q.out index 116744a29e..a140a0211f 100644 --- ql/src/test/results/clientpositive/groupby1_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby1_map_skew.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby1_noskew.q.out ql/src/test/results/clientpositive/groupby1_noskew.q.out index 98c0d3c28e..702ea3e701 100644 --- ql/src/test/results/clientpositive/groupby1_noskew.q.out +++ ql/src/test/results/clientpositive/groupby1_noskew.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby2_map.q.out ql/src/test/results/clientpositive/groupby2_map.q.out index 0dcd8109f1..427590d2ba 100644 --- ql/src/test/results/clientpositive/groupby2_map.q.out +++ ql/src/test/results/clientpositive/groupby2_map.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out index 64477dbfd2..0ab198501d 100644 --- ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) @@ -175,7 +176,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.key,1,1)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_map_skew.q.out ql/src/test/results/clientpositive/groupby2_map_skew.q.out index 813ae5cb26..bb5a09f4b3 100644 --- ql/src/test/results/clientpositive/groupby2_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby2_map_skew.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_noskew.q.out ql/src/test/results/clientpositive/groupby2_noskew.q.out index 5192db3966..782a1a2a75 100644 --- ql/src/test/results/clientpositive/groupby2_noskew.q.out +++ ql/src/test/results/clientpositive/groupby2_noskew.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out index 1c24213fba..9a070aca3f 100644 --- ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby2_noskew_multi_distinct.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby3.q.out ql/src/test/results/clientpositive/groupby3.q.out index 2ebeae450b..2f2b533d8a 100644 --- ql/src/test/results/clientpositive/groupby3.q.out +++ ql/src/test/results/clientpositive/groupby3.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_map.q.out ql/src/test/results/clientpositive/groupby3_map.q.out index 07c122e2d9..1bbf23dd5b 100644 --- ql/src/test/results/clientpositive/groupby3_map.q.out +++ ql/src/test/results/clientpositive/groupby3_map.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out index a4501f7f7a..248a31746c 100644 --- ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_map_skew.q.out ql/src/test/results/clientpositive/groupby3_map_skew.q.out index e02bdeb904..8ba3a5865a 100644 --- ql/src/test/results/clientpositive/groupby3_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby3_map_skew.q.out @@ -112,7 +112,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_noskew.q.out ql/src/test/results/clientpositive/groupby3_noskew.q.out index 624fd2db54..5c520bbdd1 100644 --- ql/src/test/results/clientpositive/groupby3_noskew.q.out +++ ql/src/test/results/clientpositive/groupby3_noskew.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out index a1d403d6cf..c5d121b4f9 100644 --- ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby3_noskew_multi_distinct.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/groupby4.q.out ql/src/test/results/clientpositive/groupby4.q.out index 3f77e47bd1..e8e7b88412 100644 --- ql/src/test/results/clientpositive/groupby4.q.out +++ ql/src/test/results/clientpositive/groupby4.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby4_map.q.out ql/src/test/results/clientpositive/groupby4_map.q.out index 97915e76f8..647242aaa4 100644 --- ql/src/test/results/clientpositive/groupby4_map.q.out +++ ql/src/test/results/clientpositive/groupby4_map.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby4_map_skew.q.out ql/src/test/results/clientpositive/groupby4_map_skew.q.out index ae83f7ac65..0857f22bbc 100644 --- ql/src/test/results/clientpositive/groupby4_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby4_map_skew.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby4_noskew.q.out ql/src/test/results/clientpositive/groupby4_noskew.q.out index c7db0d7016..5827ef06ba 100644 --- ql/src/test/results/clientpositive/groupby4_noskew.q.out +++ ql/src/test/results/clientpositive/groupby4_noskew.q.out @@ -61,7 +61,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby5.q.out ql/src/test/results/clientpositive/groupby5.q.out index 9bf01ee51b..995d30e9a1 100644 --- ql/src/test/results/clientpositive/groupby5.q.out +++ ql/src/test/results/clientpositive/groupby5.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/groupby5_map.q.out ql/src/test/results/clientpositive/groupby5_map.q.out index 5fbd3d7dad..393c4e2d54 100644 --- ql/src/test/results/clientpositive/groupby5_map.q.out +++ ql/src/test/results/clientpositive/groupby5_map.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby5_map_skew.q.out ql/src/test/results/clientpositive/groupby5_map_skew.q.out index 60b010b501..67767f5153 100644 --- ql/src/test/results/clientpositive/groupby5_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby5_map_skew.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/groupby5_noskew.q.out ql/src/test/results/clientpositive/groupby5_noskew.q.out index 612a0f6112..dfecdb5cef 100644 --- ql/src/test/results/clientpositive/groupby5_noskew.q.out +++ ql/src/test/results/clientpositive/groupby5_noskew.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/groupby6.q.out ql/src/test/results/clientpositive/groupby6.q.out index b79022405b..afc23ecf9f 100644 --- ql/src/test/results/clientpositive/groupby6.q.out +++ ql/src/test/results/clientpositive/groupby6.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby6_map.q.out ql/src/test/results/clientpositive/groupby6_map.q.out index 4ba3772298..c0c8841adc 100644 --- ql/src/test/results/clientpositive/groupby6_map.q.out +++ ql/src/test/results/clientpositive/groupby6_map.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby6_map_skew.q.out ql/src/test/results/clientpositive/groupby6_map_skew.q.out index 5141c0d9b3..5bb8695de6 100644 --- ql/src/test/results/clientpositive/groupby6_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby6_map_skew.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby6_noskew.q.out ql/src/test/results/clientpositive/groupby6_noskew.q.out index fd796c7b74..0cafbd1256 100644 --- ql/src/test/results/clientpositive/groupby6_noskew.q.out +++ ql/src/test/results/clientpositive/groupby6_noskew.q.out @@ -61,7 +61,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/groupby7_map.q.out ql/src/test/results/clientpositive/groupby7_map.q.out index 0ef29cd29f..b7b2c8ce5e 100644 --- ql/src/test/results/clientpositive/groupby7_map.q.out +++ ql/src/test/results/clientpositive/groupby7_map.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -145,7 +146,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out index 7c3b033a62..6f88027bc4 100644 --- ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby7_map_multi_single_reducer.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -112,7 +113,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_map_skew.q.out ql/src/test/results/clientpositive/groupby7_map_skew.q.out index 4bfa52ed89..17eb283633 100644 --- ql/src/test/results/clientpositive/groupby7_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby7_map_skew.q.out @@ -128,7 +128,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -195,7 +196,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_noskew.q.out ql/src/test/results/clientpositive/groupby7_noskew.q.out index 6178f58f7e..2303b3366b 100644 --- ql/src/test/results/clientpositive/groupby7_noskew.q.out +++ ql/src/test/results/clientpositive/groupby7_noskew.q.out @@ -90,7 +90,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -133,7 +134,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out index f38c428781..ff29aec5ef 100644 --- ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -164,7 +165,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key ORDER BY SRC.key limit 10 diff --git ql/src/test/results/clientpositive/groupby8.q.out ql/src/test/results/clientpositive/groupby8.q.out index 1856a9252b..826064cd53 100644 --- ql/src/test/results/clientpositive/groupby8.q.out +++ ql/src/test/results/clientpositive/groupby8.q.out @@ -115,7 +115,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -181,7 +182,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -936,7 +938,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -1002,7 +1005,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby8_map.q.out ql/src/test/results/clientpositive/groupby8_map.q.out index f683a8ba71..abe813c2b6 100644 --- ql/src/test/results/clientpositive/groupby8_map.q.out +++ ql/src/test/results/clientpositive/groupby8_map.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -111,7 +112,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby8_map_skew.q.out ql/src/test/results/clientpositive/groupby8_map_skew.q.out index 5e60d3e924..a4de8b4e06 100644 --- ql/src/test/results/clientpositive/groupby8_map_skew.q.out +++ ql/src/test/results/clientpositive/groupby8_map_skew.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -193,7 +194,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby8_noskew.q.out ql/src/test/results/clientpositive/groupby8_noskew.q.out index f683a8ba71..abe813c2b6 100644 --- ql/src/test/results/clientpositive/groupby8_noskew.q.out +++ ql/src/test/results/clientpositive/groupby8_noskew.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -111,7 +112,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby9.q.out ql/src/test/results/clientpositive/groupby9.q.out index 15ea1857c0..28032e324f 100644 --- ql/src/test/results/clientpositive/groupby9.q.out +++ ql/src/test/results/clientpositive/groupby9.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -143,7 +144,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -885,7 +887,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -927,7 +930,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -1669,7 +1673,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -1711,7 +1716,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -2454,7 +2460,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -2497,7 +2504,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -3239,7 +3247,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -3281,7 +3290,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/groupby_complex_types.q.out ql/src/test/results/clientpositive/groupby_complex_types.q.out index b16a4ade68..107eea3a1d 100644 --- ql/src/test/results/clientpositive/groupby_complex_types.q.out +++ ql/src/test/results/clientpositive/groupby_complex_types.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -166,7 +167,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce @@ -205,7 +207,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) diff --git ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out index caa5395031..bfddf74adc 100644 --- ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -191,7 +192,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10 diff --git ql/src/test/results/clientpositive/groupby_cube1.q.out ql/src/test/results/clientpositive/groupby_cube1.q.out index 9acccf3b45..38243592b4 100644 --- ql/src/test/results/clientpositive/groupby_cube1.q.out +++ ql/src/test/results/clientpositive/groupby_cube1.q.out @@ -652,7 +652,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -720,7 +721,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with cube diff --git ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out index 1e13288c9a..0f77b3d1ca 100644 --- ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out +++ ql/src/test/results/clientpositive/groupby_cube_multi_gby.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.t1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -142,5 +143,6 @@ STAGE PLANS: name: default.t2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/groupby_duplicate_key.q.out ql/src/test/results/clientpositive/groupby_duplicate_key.q.out index 6e19930927..d1d7f8d269 100644 --- ql/src/test/results/clientpositive/groupby_duplicate_key.q.out +++ ql/src/test/results/clientpositive/groupby_duplicate_key.q.out @@ -143,7 +143,8 @@ STAGE PLANS: name: default.dummy Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table dummy as select distinct key, "X" as dummy1, "X" as dummy2 from src tablesample (10 rows) diff --git ql/src/test/results/clientpositive/groupby_map_ppr.q.out ql/src/test/results/clientpositive/groupby_map_ppr.q.out index 0e5b394215..e0952c8cdf 100644 --- ql/src/test/results/clientpositive/groupby_map_ppr.q.out +++ ql/src/test/results/clientpositive/groupby_map_ppr.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out index dbcef22473..3c53a58455 100644 --- ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out index 5f02b04c38..6606edb9a1 100644 --- ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_multi_insert_common_distinct.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -143,7 +144,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table dest1 select key, count(distinct value) group by key diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out index 256784d3d7..ca1ebd7586 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out @@ -151,7 +151,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -164,7 +165,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -177,7 +179,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) @@ -389,7 +392,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -402,7 +406,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -415,7 +420,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-9 Map Reduce @@ -510,7 +516,8 @@ STAGE PLANS: name: default.dest_h2 Stage: Stage-11 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Move Operator @@ -523,7 +530,8 @@ STAGE PLANS: name: default.dest_h3 Stage: Stage-12 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out index 012b2114b4..daa047db85 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -117,7 +118,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out index 07997d2125..496580b969 100644 --- ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out +++ ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -129,7 +130,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -282,7 +284,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -295,7 +298,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -448,7 +452,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -461,7 +466,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -614,7 +620,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -627,7 +634,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 diff --git ql/src/test/results/clientpositive/groupby_position.q.out ql/src/test/results/clientpositive/groupby_position.q.out index 12c5f3fde9..16924f3459 100644 --- ql/src/test/results/clientpositive/groupby_position.q.out +++ ql/src/test/results/clientpositive/groupby_position.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -141,7 +142,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 @@ -289,7 +291,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -331,7 +334,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 diff --git ql/src/test/results/clientpositive/groupby_ppr.q.out ql/src/test/results/clientpositive/groupby_ppr.q.out index e645f5f598..8a18187eb7 100644 --- ql/src/test/results/clientpositive/groupby_ppr.q.out +++ ql/src/test/results/clientpositive/groupby_ppr.q.out @@ -221,7 +221,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out index f300095bd7..6e4501d022 100644 --- ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/groupby_ppr_multi_distinct.q.out @@ -221,7 +221,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src @@ -482,7 +483,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/groupby_rollup1.q.out ql/src/test/results/clientpositive/groupby_rollup1.q.out index e050d0af29..b7e93d9c25 100644 --- ql/src/test/results/clientpositive/groupby_rollup1.q.out +++ ql/src/test/results/clientpositive/groupby_rollup1.q.out @@ -497,7 +497,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -565,7 +566,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with rollup diff --git ql/src/test/results/clientpositive/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_1_23.q.out index 1f12c52f6a..80ec75aff5 100644 --- ql/src/test/results/clientpositive/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_1_23.q.out @@ -200,7 +200,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -585,7 +586,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -781,7 +783,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1155,7 +1158,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1537,7 +1541,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1923,7 +1928,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -2123,7 +2129,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -2335,7 +2342,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2595,7 +2603,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -3168,7 +3177,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -3591,7 +3601,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4099,7 +4110,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4293,7 +4305,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -4678,7 +4691,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5062,7 +5076,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5453,7 +5468,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5784,7 +5800,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -5797,7 +5814,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -5942,7 +5960,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -5955,7 +5974,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/groupby_sort_2.q.out ql/src/test/results/clientpositive/groupby_sort_2.q.out index bb6273e37d..9d4143660b 100644 --- ql/src/test/results/clientpositive/groupby_sort_2.q.out +++ ql/src/test/results/clientpositive/groupby_sort_2.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 SELECT val, count(1) FROM T1 GROUP BY val diff --git ql/src/test/results/clientpositive/groupby_sort_3.q.out ql/src/test/results/clientpositive/groupby_sort_3.q.out index 2dae25d085..67b53223a8 100644 --- ql/src/test/results/clientpositive/groupby_sort_3.q.out +++ ql/src/test/results/clientpositive/groupby_sort_3.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -237,7 +238,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/groupby_sort_4.q.out ql/src/test/results/clientpositive/groupby_sort_4.q.out index 70e8ac7d8d..0b269783a3 100644 --- ql/src/test/results/clientpositive/groupby_sort_4.q.out +++ ql/src/test/results/clientpositive/groupby_sort_4.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 SELECT key, count(1) FROM T1 GROUP BY key @@ -204,7 +205,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 SELECT key, val, count(1) FROM T1 GROUP BY key, val diff --git ql/src/test/results/clientpositive/groupby_sort_5.q.out ql/src/test/results/clientpositive/groupby_sort_5.q.out index db18928faa..41e1782724 100644 --- ql/src/test/results/clientpositive/groupby_sort_5.q.out +++ ql/src/test/results/clientpositive/groupby_sort_5.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -265,7 +266,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -436,7 +438,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 SELECT key, count(1) FROM T1 GROUP BY key diff --git ql/src/test/results/clientpositive/groupby_sort_6.q.out ql/src/test/results/clientpositive/groupby_sort_6.q.out index 60019e7f99..51ce026026 100644 --- ql/src/test/results/clientpositive/groupby_sort_6.q.out +++ ql/src/test/results/clientpositive/groupby_sort_6.q.out @@ -132,7 +132,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -282,7 +283,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -364,10 +366,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.t1 numFiles 1 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct t1 { string key, string val} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -470,7 +470,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 diff --git ql/src/test/results/clientpositive/groupby_sort_7.q.out ql/src/test/results/clientpositive/groupby_sort_7.q.out index 9d535e25b0..c9746c3126 100644 --- ql/src/test/results/clientpositive/groupby_sort_7.q.out +++ ql/src/test/results/clientpositive/groupby_sort_7.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out index fba8adbd49..3b3e2276ca 100644 --- ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out @@ -200,7 +200,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -654,7 +655,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -850,7 +852,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1224,7 +1227,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -1606,7 +1610,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -2061,7 +2066,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -2330,7 +2336,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -2611,7 +2618,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2871,7 +2879,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -3513,7 +3522,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -3936,7 +3946,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4582,7 +4593,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -4776,7 +4788,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5161,7 +5174,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5545,7 +5559,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -5936,7 +5951,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -6292,7 +6308,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -6305,7 +6322,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -6475,7 +6493,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -6488,7 +6507,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/groupby_sort_test_1.q.out ql/src/test/results/clientpositive/groupby_sort_test_1.q.out index d06cd7cce0..1723e75467 100644 --- ql/src/test/results/clientpositive/groupby_sort_test_1.q.out +++ ql/src/test/results/clientpositive/groupby_sort_test_1.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/hll.q.out ql/src/test/results/clientpositive/hll.q.out index 3b833844a0..3b90b56f7b 100644 --- ql/src/test/results/clientpositive/hll.q.out +++ ql/src/test/results/clientpositive/hll.q.out @@ -58,7 +58,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -67,10 +68,12 @@ STAGE PLANS: PREHOOK: query: analyze table n compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@n +PREHOOK: Output: default@n #### A masked pattern was here #### POSTHOOK: query: analyze table n compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@n +POSTHOOK: Output: default@n #### A masked pattern was here #### PREHOOK: query: desc formatted n key PREHOOK: type: DESCTABLE @@ -151,7 +154,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key Column Types: int @@ -160,10 +164,12 @@ STAGE PLANS: PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -212,10 +218,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -264,10 +272,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE @@ -342,10 +352,12 @@ POSTHOOK: Lineage: i.key EXPRESSION [(values__tmp__table__5)values__tmp__table__ PREHOOK: query: analyze table i compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@i +PREHOOK: Output: default@i #### A masked pattern was here #### POSTHOOK: query: analyze table i compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@i +POSTHOOK: Output: default@i #### A masked pattern was here #### PREHOOK: query: desc formatted i key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out index c4b9dc4c62..1fa9e94aa7 100644 --- ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out +++ ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.implicit_cast_during_insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table implicit_cast_during_insert partition (p1) select key, value, key key1 from (select * from src where key in (0,1)) q diff --git ql/src/test/results/clientpositive/index_auto_update.q.out ql/src/test/results/clientpositive/index_auto_update.q.out index e7bc0690ad..bc444cc9fb 100644 --- ql/src/test/results/clientpositive/index_auto_update.q.out +++ ql/src/test/results/clientpositive/index_auto_update.q.out @@ -135,14 +135,16 @@ STAGE PLANS: name: default.default__temp_temp_index__ Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Stage: Stage-5 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-8 Map Reduce diff --git ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out index 3d1443bde8..cba50a758d 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out @@ -486,7 +486,8 @@ STAGE PLANS: name: default.test_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out index 7fec770dc3..8ae732096a 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_grouping_operators.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: SELECT key, value, count(1) FROM src GROUP BY ROLLUP (key, value) PREHOOK: type: QUERY @@ -1510,7 +1511,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT key, value, count(1) FROM src GROUP BY key, value WITH CUBE @@ -1684,7 +1686,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT key, value, count(1) FROM src GROUP BY key, value GROUPING SETS (key, value) diff --git ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out index 041d053b85..f2218e3697 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_map_operators.q.out @@ -114,7 +114,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -293,7 +294,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT a.key, a.value FROM ( @@ -422,7 +424,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -575,7 +578,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT /*+ MAPJOIN(a) */ b.value, count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key diff --git ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out index 5086a97825..5c7659b7ba 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.test_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (ds = '2008-04-08', hr) SELECT key2, value, cast(hr as int) FROM diff --git ql/src/test/results/clientpositive/innerjoin.q.out ql/src/test/results/clientpositive/innerjoin.q.out index 99b3d856eb..741c9fbbd4 100644 --- ql/src/test/results/clientpositive/innerjoin.q.out +++ ql/src/test/results/clientpositive/innerjoin.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 INNER JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/input11.q.out ql/src/test/results/clientpositive/input11.q.out index bb22ee86df..356cef120a 100644 --- ql/src/test/results/clientpositive/input11.q.out +++ ql/src/test/results/clientpositive/input11.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input11_limit.q.out ql/src/test/results/clientpositive/input11_limit.q.out index 597554e02f..8fb7b630fc 100644 --- ql/src/test/results/clientpositive/input11_limit.q.out +++ ql/src/test/results/clientpositive/input11_limit.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 LIMIT 10 diff --git ql/src/test/results/clientpositive/input12.q.out ql/src/test/results/clientpositive/input12.q.out index cd9d52e903..8bb52c0cd1 100644 --- ql/src/test/results/clientpositive/input12.q.out +++ ql/src/test/results/clientpositive/input12.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -183,7 +184,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-11 Map Reduce @@ -238,7 +240,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-16 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-17 Map Reduce diff --git ql/src/test/results/clientpositive/input13.q.out ql/src/test/results/clientpositive/input13.q.out index c594ce3195..9b1970b7ab 100644 --- ql/src/test/results/clientpositive/input13.q.out +++ ql/src/test/results/clientpositive/input13.q.out @@ -153,7 +153,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -205,7 +206,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-11 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-12 Map Reduce @@ -260,7 +262,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-17 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-18 Map Reduce diff --git ql/src/test/results/clientpositive/input14.q.out ql/src/test/results/clientpositive/input14.q.out index af04a9896d..d8455267eb 100644 --- ql/src/test/results/clientpositive/input14.q.out +++ ql/src/test/results/clientpositive/input14.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input14_limit.q.out ql/src/test/results/clientpositive/input14_limit.q.out index 9870ad53b4..f62ad051c7 100644 --- ql/src/test/results/clientpositive/input14_limit.q.out +++ ql/src/test/results/clientpositive/input14_limit.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input17.q.out ql/src/test/results/clientpositive/input17.q.out index 057a92d163..316c9c6c6b 100644 --- ql/src/test/results/clientpositive/input17.q.out +++ ql/src/test/results/clientpositive/input17.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src_thrift diff --git ql/src/test/results/clientpositive/input18.q.out ql/src/test/results/clientpositive/input18.q.out index b341510502..4eb75ff55e 100644 --- ql/src/test/results/clientpositive/input18.q.out +++ ql/src/test/results/clientpositive/input18.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input1_limit.q.out ql/src/test/results/clientpositive/input1_limit.q.out index 0ca1552ef2..649e8fb94b 100644 --- ql/src/test/results/clientpositive/input1_limit.q.out +++ ql/src/test/results/clientpositive/input1_limit.q.out @@ -102,7 +102,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -145,7 +146,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 LIMIT 10 diff --git ql/src/test/results/clientpositive/input20.q.out ql/src/test/results/clientpositive/input20.q.out index 9587445f82..051e94ecf3 100644 --- ql/src/test/results/clientpositive/input20.q.out +++ ql/src/test/results/clientpositive/input20.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input30.q.out ql/src/test/results/clientpositive/input30.q.out index 478cea1861..90c9732117 100644 --- ql/src/test/results/clientpositive/input30.q.out +++ ql/src/test/results/clientpositive/input30.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.tst_dest30 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest30 select count(1) from src @@ -159,7 +160,8 @@ STAGE PLANS: name: default.dest30 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest30 select count(1) from src diff --git ql/src/test/results/clientpositive/input31.q.out ql/src/test/results/clientpositive/input31.q.out index ea2c8f95b3..d3c2c6af09 100644 --- ql/src/test/results/clientpositive/input31.q.out +++ ql/src/test/results/clientpositive/input31.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.tst_dest31 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest31 select count(1) from srcbucket diff --git ql/src/test/results/clientpositive/input32.q.out ql/src/test/results/clientpositive/input32.q.out index d3426a8dfa..fa3b731a78 100644 --- ql/src/test/results/clientpositive/input32.q.out +++ ql/src/test/results/clientpositive/input32.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.tst_dest32 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest32 select count(1) from srcbucket diff --git ql/src/test/results/clientpositive/input33.q.out ql/src/test/results/clientpositive/input33.q.out index 4be5fc5d61..c24ac6e41e 100644 --- ql/src/test/results/clientpositive/input33.q.out +++ ql/src/test/results/clientpositive/input33.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/input34.q.out ql/src/test/results/clientpositive/input34.q.out index 72f66c3f17..841f0af16f 100644 --- ql/src/test/results/clientpositive/input34.q.out +++ ql/src/test/results/clientpositive/input34.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input35.q.out ql/src/test/results/clientpositive/input35.q.out index 8b869918e1..76a014eed9 100644 --- ql/src/test/results/clientpositive/input35.q.out +++ ql/src/test/results/clientpositive/input35.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input36.q.out ql/src/test/results/clientpositive/input36.q.out index 76921ad6b8..f75f8fda5a 100644 --- ql/src/test/results/clientpositive/input36.q.out +++ ql/src/test/results/clientpositive/input36.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input38.q.out ql/src/test/results/clientpositive/input38.q.out index 0c4e81d0ef..41a0dbd420 100644 --- ql/src/test/results/clientpositive/input38.q.out +++ ql/src/test/results/clientpositive/input38.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input3_limit.q.out ql/src/test/results/clientpositive/input3_limit.q.out index 07b84ed1c1..d187ee2818 100644 --- ql/src/test/results/clientpositive/input3_limit.q.out +++ ql/src/test/results/clientpositive/input3_limit.q.out @@ -107,7 +107,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE T2 SELECT * FROM (SELECT * FROM T1 DISTRIBUTE BY key) T ORDER BY key, value LIMIT 20 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/input4.q.out ql/src/test/results/clientpositive/input4.q.out index bbcce4ae7e..151aa0c4c1 100644 --- ql/src/test/results/clientpositive/input4.q.out +++ ql/src/test/results/clientpositive/input4.q.out @@ -28,7 +28,8 @@ STAGE PLANS: name: default.input4 Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE INPUT4 PREHOOK: type: LOAD @@ -44,7 +45,7 @@ PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN FORMATTED SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias POSTHOOK: type: QUERY -{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","Statistics:":"Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 1 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"input4alias","Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"value (type: string), key (type: string)","outputColumnNames:":["_col0","_col1"],"Statistics:":"Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE","OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} PREHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias PREHOOK: type: QUERY PREHOOK: Input: default@input4 diff --git ql/src/test/results/clientpositive/input5.q.out ql/src/test/results/clientpositive/input5.q.out index a39952878d..75a827c5da 100644 --- ql/src/test/results/clientpositive/input5.q.out +++ ql/src/test/results/clientpositive/input5.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src_thrift diff --git ql/src/test/results/clientpositive/input6.q.out ql/src/test/results/clientpositive/input6.q.out index 3d1a815cf6..35e746ae8c 100644 --- ql/src/test/results/clientpositive/input6.q.out +++ ql/src/test/results/clientpositive/input6.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input7.q.out ql/src/test/results/clientpositive/input7.q.out index 0545b1f774..281f6d4a09 100644 --- ql/src/test/results/clientpositive/input7.q.out +++ ql/src/test/results/clientpositive/input7.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input8.q.out ql/src/test/results/clientpositive/input8.q.out index d76fc2bc02..e49c7e17f7 100644 --- ql/src/test/results/clientpositive/input8.q.out +++ ql/src/test/results/clientpositive/input8.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input9.q.out ql/src/test/results/clientpositive/input9.q.out index af752e0af9..8df7fb760e 100644 --- ql/src/test/results/clientpositive/input9.q.out +++ ql/src/test/results/clientpositive/input9.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_columnarserde.q.out ql/src/test/results/clientpositive/input_columnarserde.q.out index afa0e28d17..95d0e6e7ec 100644 --- ql/src/test/results/clientpositive/input_columnarserde.q.out +++ ql/src/test/results/clientpositive/input_columnarserde.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.input_columnarserde Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE input_columnarserde SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/input_dynamicserde.q.out ql/src/test/results/clientpositive/input_dynamicserde.q.out index 30493be357..a117873164 100644 --- ql/src/test/results/clientpositive/input_dynamicserde.q.out +++ ql/src/test/results/clientpositive/input_dynamicserde.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_lazyserde.q.out ql/src/test/results/clientpositive/input_lazyserde.q.out index 64dc6c14a4..473b5fe4e6 100644 --- ql/src/test/results/clientpositive/input_lazyserde.q.out +++ ql/src/test/results/clientpositive/input_lazyserde.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/input_lazyserde2.q.out ql/src/test/results/clientpositive/input_lazyserde2.q.out index 82c72dbed8..d6583341bf 100644 --- ql/src/test/results/clientpositive/input_lazyserde2.q.out +++ ql/src/test/results/clientpositive/input_lazyserde2.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 diff --git ql/src/test/results/clientpositive/input_part1.q.out ql/src/test/results/clientpositive/input_part1.q.out index a68544671f..9a9b1921c9 100644 --- ql/src/test/results/clientpositive/input_part1.q.out +++ ql/src/test/results/clientpositive/input_part1.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/input_part10.q.out ql/src/test/results/clientpositive/input_part10.q.out index 5ec24a79cf..e8ad15b914 100644 --- ql/src/test/results/clientpositive/input_part10.q.out +++ ql/src/test/results/clientpositive/input_part10.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.part_special Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE part_special PARTITION(ds='2008 04 08', ts = '10:11:12=455') SELECT 1, 2 FROM src LIMIT 1 diff --git ql/src/test/results/clientpositive/input_part2.q.out ql/src/test/results/clientpositive/input_part2.q.out index 0211d506da..6942b23984 100644 --- ql/src/test/results/clientpositive/input_part2.q.out +++ ql/src/test/results/clientpositive/input_part2.q.out @@ -270,7 +270,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -491,7 +492,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 diff --git ql/src/test/results/clientpositive/input_part5.q.out ql/src/test/results/clientpositive/input_part5.q.out index c6ae2fd58d..7da77fb3cd 100644 --- ql/src/test/results/clientpositive/input_part5.q.out +++ ql/src/test/results/clientpositive/input_part5.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_testsequencefile.q.out ql/src/test/results/clientpositive/input_testsequencefile.q.out index 60aaf83df5..c248d03bfc 100644 --- ql/src/test/results/clientpositive/input_testsequencefile.q.out +++ ql/src/test/results/clientpositive/input_testsequencefile.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest4_sequencefile Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_testxpath.q.out ql/src/test/results/clientpositive/input_testxpath.q.out index e07628aaea..a68a5003fe 100644 --- ql/src/test/results/clientpositive/input_testxpath.q.out +++ ql/src/test/results/clientpositive/input_testxpath.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/input_testxpath2.q.out ql/src/test/results/clientpositive/input_testxpath2.q.out index a0baccf925..ed45157141 100644 --- ql/src/test/results/clientpositive/input_testxpath2.q.out +++ ql/src/test/results/clientpositive/input_testxpath2.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert1.q.out ql/src/test/results/clientpositive/insert1.q.out index 5dffac7647..aeb89eb9b7 100644 --- ql/src/test/results/clientpositive/insert1.q.out +++ ql/src/test/results/clientpositive/insert1.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -170,7 +171,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -273,7 +275,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -362,7 +365,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -479,7 +483,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -531,7 +536,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-10 Map Reduce diff --git ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out index 49c1269cc1..a9378f81e5 100644 --- ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out +++ ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out @@ -90,7 +90,8 @@ STAGE PLANS: name: default.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE destinTable PARTITION (ds='2011-11-11', hr='11') if not exists SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5 @@ -220,7 +221,8 @@ STAGE PLANS: name: default.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE destinTable SELECT one,two FROM sourceTable WHERE ds='2011-11-11' AND hr='11' order by one desc, two desc limit 5 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out index b5f28d289a..c8bfdc6bb2 100644 --- ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out +++ ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out @@ -100,7 +100,8 @@ STAGE PLANS: name: db2.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11') SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5 @@ -188,7 +189,8 @@ STAGE PLANS: name: db2.destintable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE db2.destinTable PARTITION (ds='2011-11-11') SELECT one,two FROM db1.sourceTable WHERE ds='2011-11-11' order by one desc, two desc limit 5 diff --git ql/src/test/results/clientpositive/insert_into1.q.out ql/src/test/results/clientpositive/insert_into1.q.out index da863a7185..3d1438a037 100644 --- ql/src/test/results/clientpositive/insert_into1.q.out +++ ql/src/test/results/clientpositive/insert_into1.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -176,7 +177,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -284,7 +286,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 10 PREHOOK: type: QUERY @@ -389,7 +392,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -486,7 +490,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_into2.q.out ql/src/test/results/clientpositive/insert_into2.q.out index 46fab7b416..90b409cbb1 100644 --- ql/src/test/results/clientpositive/insert_into2.q.out +++ ql/src/test/results/clientpositive/insert_into2.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into2 PARTITION (ds='1') SELECT * FROM src order by key limit 100 PREHOOK: type: QUERY @@ -223,7 +224,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 100 @@ -341,7 +343,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 50 diff --git ql/src/test/results/clientpositive/insert_into3.q.out ql/src/test/results/clientpositive/insert_into3.q.out index ae7523b217..4bda7d9514 100644 --- ql/src/test/results/clientpositive/insert_into3.q.out +++ ql/src/test/results/clientpositive/insert_into3.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -137,7 +138,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 @@ -259,7 +261,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -302,7 +305,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 diff --git ql/src/test/results/clientpositive/insert_into4.q.out ql/src/test/results/clientpositive/insert_into4.q.out index bb4e5571a6..931ae3d7be 100644 --- ql/src/test/results/clientpositive/insert_into4.q.out +++ ql/src/test/results/clientpositive/insert_into4.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.insert_into4a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY @@ -167,7 +168,8 @@ STAGE PLANS: name: default.insert_into4a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into4a SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY @@ -246,7 +248,8 @@ STAGE PLANS: name: default.insert_into4b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_into5.q.out ql/src/test/results/clientpositive/insert_into5.q.out index 7b471f4433..5e52e49263 100644 --- ql/src/test/results/clientpositive/insert_into5.q.out +++ ql/src/test/results/clientpositive/insert_into5.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.insert_into5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into5a SELECT 1, 'one' FROM src LIMIT 10 PREHOOK: type: QUERY @@ -157,7 +158,8 @@ STAGE PLANS: name: default.insert_into5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -270,7 +272,8 @@ STAGE PLANS: name: default.insert_into5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -385,7 +388,8 @@ STAGE PLANS: name: default.insert_into5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_into6.q.out ql/src/test/results/clientpositive/insert_into6.q.out index d93a167a74..964e259771 100644 --- ql/src/test/results/clientpositive/insert_into6.q.out +++ ql/src/test/results/clientpositive/insert_into6.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.insert_into6a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into6a PARTITION (ds='1') SELECT * FROM src LIMIT 150 PREHOOK: type: QUERY @@ -183,7 +184,8 @@ STAGE PLANS: name: default.insert_into6b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out index 33c785ac47..310b7db68f 100644 --- ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out +++ ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out @@ -166,9 +166,8 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {} numFiles 1 - numRows 0 - rawDataSize 0 totalSize 295483 transactional true #### A masked pattern was here #### @@ -358,9 +357,8 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {} numFiles 1 - numRows 0 - rawDataSize 0 totalSize 1554 transactional true #### A masked pattern was here #### @@ -479,9 +477,8 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {} numFiles 2 - numRows 0 - rawDataSize 0 totalSize 3109 transactional true #### A masked pattern was here #### @@ -598,9 +595,8 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + COLUMN_STATS_ACCURATE {} numFiles 3 - numRows 0 - rawDataSize 0 totalSize 298592 transactional true #### A masked pattern was here #### @@ -823,8 +819,6 @@ Table: sp #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -926,9 +920,8 @@ Database: default Table: sp #### A masked pattern was here #### Partition Parameters: + COLUMN_STATS_ACCURATE {} numFiles 2 - numRows 0 - rawDataSize 0 totalSize 5820 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out index c2732c8896..4c020fd26e 100644 --- ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out +++ ql/src/test/results/clientpositive/insertoverwrite_bucket.q.out @@ -142,7 +142,8 @@ STAGE PLANS: name: default.temp1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TABLE temp2 ( @@ -221,7 +222,8 @@ STAGE PLANS: name: default.temp2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: select * from bucketoutput1 a join bucketoutput2 b on (a.data=b.data) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/join14.q.out ql/src/test/results/clientpositive/join14.q.out index 66e42f1992..10b4e1fd8b 100644 --- ql/src/test/results/clientpositive/join14.q.out +++ ql/src/test/results/clientpositive/join14.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/join17.q.out ql/src/test/results/clientpositive/join17.q.out index f9edc792eb..6c3e5fe40f 100644 --- ql/src/test/results/clientpositive/join17.q.out +++ ql/src/test/results/clientpositive/join17.q.out @@ -192,7 +192,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/join2.q.out ql/src/test/results/clientpositive/join2.q.out index e3d26a29a6..b2067421ec 100644 --- ql/src/test/results/clientpositive/join2.q.out +++ ql/src/test/results/clientpositive/join2.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/join25.q.out ql/src/test/results/clientpositive/join25.q.out index 5ad95c507b..8ed420bbaa 100644 --- ql/src/test/results/clientpositive/join25.q.out +++ ql/src/test/results/clientpositive/join25.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/join26.q.out ql/src/test/results/clientpositive/join26.q.out index b41fd8efe1..134aa523d8 100644 --- ql/src/test/results/clientpositive/join26.q.out +++ ql/src/test/results/clientpositive/join26.q.out @@ -313,7 +313,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join27.q.out ql/src/test/results/clientpositive/join27.q.out index 8b43f3f32a..2e78d2ad22 100644 --- ql/src/test/results/clientpositive/join27.q.out +++ ql/src/test/results/clientpositive/join27.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/join28.q.out ql/src/test/results/clientpositive/join28.q.out index 309bdcd200..c4b122fcb6 100644 --- ql/src/test/results/clientpositive/join28.q.out +++ ql/src/test/results/clientpositive/join28.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq.key1, z.value diff --git ql/src/test/results/clientpositive/join29.q.out ql/src/test/results/clientpositive/join29.q.out index ef02385fe1..d394eda3d4 100644 --- ql/src/test/results/clientpositive/join29.q.out +++ ql/src/test/results/clientpositive/join29.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-9 Map Reduce Local Work diff --git ql/src/test/results/clientpositive/join3.q.out ql/src/test/results/clientpositive/join3.q.out index fb378f403a..8c4cfe2c37 100644 --- ql/src/test/results/clientpositive/join3.q.out +++ ql/src/test/results/clientpositive/join3.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/join30.q.out ql/src/test/results/clientpositive/join30.q.out index f06c70ae40..cdd54458db 100644 --- ql/src/test/results/clientpositive/join30.q.out +++ ql/src/test/results/clientpositive/join30.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/join31.q.out ql/src/test/results/clientpositive/join31.q.out index e055b5d2fa..7b0cbdc64e 100644 --- ql/src/test/results/clientpositive/join31.q.out +++ ql/src/test/results/clientpositive/join31.q.out @@ -145,7 +145,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq1.key, count(1) as cnt diff --git ql/src/test/results/clientpositive/join32.q.out ql/src/test/results/clientpositive/join32.q.out index 176989caff..30a5ba9195 100644 --- ql/src/test/results/clientpositive/join32.q.out +++ ql/src/test/results/clientpositive/join32.q.out @@ -366,7 +366,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join33.q.out ql/src/test/results/clientpositive/join33.q.out index 176989caff..30a5ba9195 100644 --- ql/src/test/results/clientpositive/join33.q.out +++ ql/src/test/results/clientpositive/join33.q.out @@ -366,7 +366,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join34.q.out ql/src/test/results/clientpositive/join34.q.out index 67599bc991..072a20b62d 100644 --- ql/src/test/results/clientpositive/join34.q.out +++ ql/src/test/results/clientpositive/join34.q.out @@ -309,7 +309,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/join35.q.out ql/src/test/results/clientpositive/join35.q.out index ade6646caa..f8fd4387a9 100644 --- ql/src/test/results/clientpositive/join35.q.out +++ ql/src/test/results/clientpositive/join35.q.out @@ -393,7 +393,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 diff --git ql/src/test/results/clientpositive/join36.q.out ql/src/test/results/clientpositive/join36.q.out index 43a091f629..0fd4d918e6 100644 --- ql/src/test/results/clientpositive/join36.q.out +++ ql/src/test/results/clientpositive/join36.q.out @@ -133,7 +133,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt diff --git ql/src/test/results/clientpositive/join37.q.out ql/src/test/results/clientpositive/join37.q.out index b0a2ee3dfa..c8bef838f4 100644 --- ql/src/test/results/clientpositive/join37.q.out +++ ql/src/test/results/clientpositive/join37.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/join39.q.out ql/src/test/results/clientpositive/join39.q.out index c656762675..93ebdb85c7 100644 --- ql/src/test/results/clientpositive/join39.q.out +++ ql/src/test/results/clientpositive/join39.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value diff --git ql/src/test/results/clientpositive/join4.q.out ql/src/test/results/clientpositive/join4.q.out index f11d9951ed..d4e83c444f 100644 --- ql/src/test/results/clientpositive/join4.q.out +++ ql/src/test/results/clientpositive/join4.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join5.q.out ql/src/test/results/clientpositive/join5.q.out index 99cfe1ad17..8cfe40271a 100644 --- ql/src/test/results/clientpositive/join5.q.out +++ ql/src/test/results/clientpositive/join5.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join6.q.out ql/src/test/results/clientpositive/join6.q.out index ce2ec8b234..8d6578afc6 100644 --- ql/src/test/results/clientpositive/join6.q.out +++ ql/src/test/results/clientpositive/join6.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join7.q.out ql/src/test/results/clientpositive/join7.q.out index d8adc39e06..f1af54a7b4 100644 --- ql/src/test/results/clientpositive/join7.q.out +++ ql/src/test/results/clientpositive/join7.q.out @@ -138,7 +138,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join8.q.out ql/src/test/results/clientpositive/join8.q.out index 378482439d..ce5a5d43b4 100644 --- ql/src/test/results/clientpositive/join8.q.out +++ ql/src/test/results/clientpositive/join8.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/join9.q.out ql/src/test/results/clientpositive/join9.q.out index a96f341c78..58f8d7dfbe 100644 --- ql/src/test/results/clientpositive/join9.q.out +++ ql/src/test/results/clientpositive/join9.q.out @@ -240,7 +240,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/join_map_ppr.q.out ql/src/test/results/clientpositive/join_map_ppr.q.out index a4d414089e..6eb9889f90 100644 --- ql/src/test/results/clientpositive/join_map_ppr.q.out +++ ql/src/test/results/clientpositive/join_map_ppr.q.out @@ -225,7 +225,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-5 @@ -803,7 +804,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-5 diff --git ql/src/test/results/clientpositive/join_star.q.out ql/src/test/results/clientpositive/join_star.q.out index 1215fbbc91..cbb7a03d6a 100644 --- ql/src/test/results/clientpositive/join_star.q.out +++ ql/src/test/results/clientpositive/join_star.q.out @@ -146,14 +146,14 @@ STAGE PLANS: $hdt$_1:dim1 TableScan alias: dim1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: f1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f1 (type: int), f2 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col2 (type: int) @@ -164,14 +164,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: fact - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: d1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: m1 (type: int), m2 (type: int), d1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -179,14 +179,14 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 1 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -243,14 +243,14 @@ STAGE PLANS: $hdt$_1:dim1 TableScan alias: dim1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: f1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f1 (type: int), f2 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col2 (type: int) @@ -258,14 +258,14 @@ STAGE PLANS: $hdt$_2:dim2 TableScan alias: dim2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: f3 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f3 (type: int), f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col3 (type: int) @@ -276,14 +276,14 @@ STAGE PLANS: Map Operator Tree: TableScan alias: fact - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (d1 is not null and d2 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: m1 (type: int), m2 (type: int), d1 (type: int), d2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -291,7 +291,7 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col5 - Statistics: Num rows: 1 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 107 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -299,14 +299,14 @@ STAGE PLANS: 0 _col3 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col7 - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col7 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -355,57 +355,57 @@ STAGE PLANS: Stage: Stage-7 Map Reduce Local Work Alias -> Map Local Tables: - $hdt$_0:dim1 + $hdt$_0:fact Fetch Operator limit: -1 - $hdt$_2:fact + $hdt$_1:dim1 Fetch Operator limit: -1 Alias -> Map Local Operator Tree: - $hdt$_0:dim1 - TableScan - alias: dim1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (f1 is not null and f2 is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: f1 (type: int), f2 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - HashTable Sink Operator - keys: - 0 _col1 (type: int) - 1 _col0 (type: int) - $hdt$_2:fact + $hdt$_0:fact TableScan alias: fact - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: d1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: m1 (type: int), m2 (type: int), d1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col0 (type: int) 1 _col2 (type: int) + $hdt$_1:dim1 + TableScan + alias: dim1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (f1 is not null and f2 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: f1 (type: int), f2 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col1 (type: int) + 1 _col0 (type: int) Stage: Stage-5 Map Reduce Map Operator Tree: TableScan alias: dim2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: f3 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f3 (type: int), f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -413,7 +413,7 @@ STAGE PLANS: 0 _col1 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 1 Data size: 17 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 17 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -421,14 +421,14 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col2 (type: int) outputColumnNames: _col1, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col4 (type: int), _col5 (type: int), _col1 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 18 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -487,11 +487,11 @@ STAGE PLANS: $hdt$_1:dim1 TableScan alias: dim1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f1 (type: int), f2 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col2 (type: int) @@ -499,11 +499,11 @@ STAGE PLANS: $hdt$_2:dim2 TableScan alias: dim2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f3 (type: int), f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col4 (type: int) @@ -514,11 +514,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: fact - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: m1 (type: int), m2 (type: int), d1 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -526,7 +526,7 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4 - Statistics: Num rows: 1 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -534,14 +534,14 @@ STAGE PLANS: 0 _col4 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col4, _col6 - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int), _col6 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -631,11 +631,11 @@ STAGE PLANS: $hdt$_1:dim1 TableScan alias: dim1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f1 (type: int), f2 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col2 (type: int) @@ -643,11 +643,11 @@ STAGE PLANS: $hdt$_2:dim2 TableScan alias: dim2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f3 (type: int), f4 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col5 (type: int) @@ -655,11 +655,11 @@ STAGE PLANS: $hdt$_3:dim3 TableScan alias: dim3 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f5 (type: int), f6 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col3 (type: int) @@ -667,11 +667,11 @@ STAGE PLANS: $hdt$_4:dim4 TableScan alias: dim4 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f7 (type: int), f8 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col9 (type: int) @@ -679,11 +679,11 @@ STAGE PLANS: $hdt$_5:dim5 TableScan alias: dim5 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f9 (type: int), f10 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col11 (type: int) @@ -691,11 +691,11 @@ STAGE PLANS: $hdt$_6:dim6 TableScan alias: dim6 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f11 (type: int), f12 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col9 (type: int) @@ -703,11 +703,11 @@ STAGE PLANS: $hdt$_7:dim7 TableScan alias: dim7 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: f13 (type: int), f14 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE HashTable Sink Operator keys: 0 _col15 (type: int) @@ -718,11 +718,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: fact - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: m1 (type: int), m2 (type: int), d1 (type: int), d2 (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -730,7 +730,7 @@ STAGE PLANS: 0 _col2 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col5 - Statistics: Num rows: 1 Data size: 107 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 107 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -738,7 +738,7 @@ STAGE PLANS: 0 _col5 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3, _col5, _col7 - Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -746,7 +746,7 @@ STAGE PLANS: 0 _col3 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col7, _col9 - Statistics: Num rows: 1 Data size: 128 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 128 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -754,7 +754,7 @@ STAGE PLANS: 0 _col9 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col7, _col9, _col11 - Statistics: Num rows: 1 Data size: 140 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 140 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -762,7 +762,7 @@ STAGE PLANS: 0 _col11 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col7, _col9, _col11, _col13 - Statistics: Num rows: 1 Data size: 154 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 154 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -770,7 +770,7 @@ STAGE PLANS: 0 _col9 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col7, _col9, _col11, _col13, _col15 - Statistics: Num rows: 1 Data size: 169 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 169 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join 0 to 1 @@ -778,14 +778,14 @@ STAGE PLANS: 0 _col15 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col5, _col7, _col9, _col11, _col13, _col15, _col17 - Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 185 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col7 (type: int), _col9 (type: int), _col11 (type: int), _col13 (type: int), _col15 (type: int), _col17 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 185 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 185 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 6 Data size: 185 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/limit_pushdown_negative.q.out ql/src/test/results/clientpositive/limit_pushdown_negative.q.out index eeb47f7fe5..7d0abfdf97 100644 --- ql/src/test/results/clientpositive/limit_pushdown_negative.q.out +++ ql/src/test/results/clientpositive/limit_pushdown_negative.q.out @@ -296,7 +296,8 @@ STAGE PLANS: name: default.dest_2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -339,5 +340,6 @@ STAGE PLANS: name: default.dest_3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/lineage1.q.out ql/src/test/results/clientpositive/lineage1.q.out index 6c8a22fa9c..7d6af3e1df 100644 --- ql/src/test/results/clientpositive/lineage1.q.out +++ ql/src/test/results/clientpositive/lineage1.q.out @@ -147,7 +147,8 @@ STAGE PLANS: name: default.dest_l1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/list_bucket_dml_1.q.out ql/src/test/results/clientpositive/list_bucket_dml_1.q.out index a664db3fee..2eca946673 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_1.q.out @@ -197,7 +197,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08' diff --git ql/src/test/results/clientpositive/list_bucket_dml_11.q.out ql/src/test/results/clientpositive/list_bucket_dml_11.q.out index bf987e13df..6b347454ac 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_11.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_11.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/list_bucket_dml_12.q.out ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index 3c80cef8fc..a0bc3e99ce 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.list_bucketing_mul_col Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/list_bucket_dml_13.q.out ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index 379a7305ff..df8717b0b3 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.list_bucketing_mul_col Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99') diff --git ql/src/test/results/clientpositive/list_bucket_dml_14.q.out ql/src/test/results/clientpositive/list_bucket_dml_14.q.out index b6b829a3c0..6b48f3dd83 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_14.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_14.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.list_bucketing Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing select * from src diff --git ql/src/test/results/clientpositive/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index 8167730752..512aeabb8c 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/list_bucket_dml_3.q.out ql/src/test/results/clientpositive/list_bucket_dml_3.q.out index aafdbb1bf5..e86bf0a829 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_3.q.out @@ -191,7 +191,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds='2008-04-08', hr='11') select key, value from srcpart where ds='2008-04-08' diff --git ql/src/test/results/clientpositive/list_bucket_dml_4.q.out ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index 36bd66697d..7f90ac5928 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') @@ -469,7 +470,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/list_bucket_dml_5.q.out ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index 23ec5f431b..64fdacb54c 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -197,7 +197,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08' diff --git ql/src/test/results/clientpositive/list_bucket_dml_6.q.out ql/src/test/results/clientpositive/list_bucket_dml_6.q.out index 1e01ca3720..32dac42591 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_6.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_6.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr) @@ -516,7 +517,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/list_bucket_dml_7.q.out ql/src/test/results/clientpositive/list_bucket_dml_7.q.out index eea69ff989..7c11d3ff8f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_7.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_7.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr) @@ -516,7 +517,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/list_bucket_dml_8.q.out ql/src/test/results/clientpositive/list_bucket_dml_8.q.out index 0215f5d4e2..db646debf6 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_8.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_8.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr) @@ -345,8 +346,6 @@ Table: list_bucketing_dynamic_part #### A masked pattern was here #### Partition Parameters: numFiles 3 - numRows 984 - rawDataSize 9488 totalSize 10586 #### A masked pattern was here #### @@ -462,10 +461,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.list_bucketing_dynamic_part numFiles 3 - numRows 984 partition_columns ds/hr partition_columns.types string:string - rawDataSize 9488 serialization.ddl struct list_bucketing_dynamic_part { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -495,16 +492,16 @@ STAGE PLANS: Processor Tree: TableScan alias: list_bucketing_dynamic_part - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 136 Basic stats: PARTIAL Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: ((key = '484') and (value = 'val_484')) (type: boolean) - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484' diff --git ql/src/test/results/clientpositive/list_bucket_dml_9.q.out ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index fdc7a3ef5b..9df21303c0 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') @@ -469,7 +470,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out index 77bea79574..7c6d55186e 100644 --- ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out +++ ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out @@ -43,22 +43,22 @@ STAGE PLANS: alias: acidtbldefault filterExpr: (a = 1) (type: boolean) buckets included: [1,] of 16 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8143 Data size: 30948 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (a = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 19 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 19 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 19 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -86,6 +86,7 @@ STAGE PLANS: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: + COLUMN_STATS_ACCURATE {} bucket_count 16 bucket_field_name a column.name.delimiter , @@ -95,8 +96,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.acidtbldefault numFiles 17 - numRows 0 - rawDataSize 0 serialization.ddl struct acidtbldefault { i32 a} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -109,6 +108,7 @@ STAGE PLANS: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: + COLUMN_STATS_ACCURATE {} bucket_count 16 bucket_field_name a column.name.delimiter , @@ -118,8 +118,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.acidtbldefault numFiles 17 - numRows 0 - rawDataSize 0 serialization.ddl struct acidtbldefault { i32 a} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde diff --git ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out index 85f005e317..af3476caa7 100644 --- ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out +++ ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out @@ -169,7 +169,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acid set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -348,7 +349,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acid where key in( '1001', '213', '43') PREHOOK: type: QUERY @@ -665,7 +667,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acidb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -845,7 +848,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acidb where key in( '1001', '213', '43') PREHOOK: type: QUERY @@ -1161,7 +1165,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acidv set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -1340,7 +1345,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acidv where key in( '1001', '213', '43') PREHOOK: type: QUERY @@ -1657,7 +1663,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update srcpart_acidvb set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11' PREHOOK: type: QUERY @@ -1837,7 +1844,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from srcpart_acidvb where key in( '1001', '213', '43') PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out index 9fbc4040ec..f93f666542 100644 --- ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out +++ ql/src/test/results/clientpositive/llap/autoColumnStats_1.q.out @@ -374,6 +374,44 @@ POSTHOOK: query: create table if not exists nzhang_part14 (key string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: desc formatted nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string + +# Partition Information +# col_name data_type comment +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numPartitions 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a @@ -399,6 +437,42 @@ POSTHOOK: Output: default@nzhang_part14@value= POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ POSTHOOK: Lineage: nzhang_part14 PARTITION(value= ).key EXPRESSION [] POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] +PREHOOK: query: desc formatted nzhang_part14 partition (value=' ') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: desc formatted nzhang_part14 partition (value=' ') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +# col_name data_type comment +key string + +# Partition Information +# col_name data_type comment +value string + +# Detailed Partition Information +Partition Value: [ ] +Database: default +Table: nzhang_part14 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} + numFiles 1 + numRows 2 + rawDataSize 4 + totalSize 6 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain select key from nzhang_part14 PREHOOK: type: QUERY POSTHOOK: query: explain select key from nzhang_part14 diff --git ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out new file mode 100644 index 0000000000..6cb51fdab9 --- /dev/null +++ ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out @@ -0,0 +1,516 @@ +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min 1 +max 1 +num_nulls 0 +distinct_count 1 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min 1 +max 2 +num_nulls 0 +distinct_count 2 +avg_col_len +max_col_len +num_trues +num_falses +bitVector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} +PREHOOK: query: drop table p +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: drop table p +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@p +POSTHOOK: query: CREATE TABLE p(insert_num int, c1 tinyint, c2 smallint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert into p values (1,22,333) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (1,22,333) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 tinyint +c2 smallint + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +PREHOOK: type: ALTERTABLE_REPLACECOLS +PREHOOK: Input: default@p +PREHOOK: Output: default@p +POSTHOOK: query: alter table p replace columns (insert_num int, c1 STRING, c2 STRING) +POSTHOOK: type: ALTERTABLE_REPLACECOLS +POSTHOOK: Input: default@p +POSTHOOK: Output: default@p +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 1 + numRows 1 + rawDataSize 8 + totalSize 9 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: insert into p values (2,11,111) +PREHOOK: type: QUERY +PREHOOK: Output: default@p +POSTHOOK: query: insert into p values (2,11,111) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@p +POSTHOOK: Lineage: p.c1 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: p.c2 SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: p.insert_num EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted p +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +# col_name data_type comment +insert_num int +c1 string +c2 string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +#### A masked pattern was here #### + numFiles 2 + numRows 2 + rawDataSize 16 + totalSize 18 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted p insert_num +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p insert_num +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name insert_num +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: desc formatted p c1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@p +POSTHOOK: query: desc formatted p c1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@p +col_name c1 +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bitVector +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} diff --git ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out index 104c25f94d..0cc0c051fd 100644 --- ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out +++ ql/src/test/results/clientpositive/llap/autoColumnStats_2.q.out @@ -747,7 +747,6 @@ Database: default Table: alter5 #### A masked pattern was here #### Partition Parameters: - COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"col1\":\"true\"}} numFiles 1 totalSize 1906 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/auto_join1.q.out ql/src/test/results/clientpositive/llap/auto_join1.q.out index 5329f84f34..bbe63e24d4 100644 --- ql/src/test/results/clientpositive/llap/auto_join1.q.out +++ ql/src/test/results/clientpositive/llap/auto_join1.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out index c6b14abd2f..80a74fbfdf 100644 --- ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/auto_smb_mapjoin_14.q.out @@ -1612,7 +1612,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1625,7 +1626,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -1850,7 +1852,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1863,7 +1866,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out index 36bfac3f4c..1d137dd84d 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out @@ -160,10 +160,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -253,10 +251,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -302,10 +298,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -468,10 +462,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -517,10 +509,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -596,10 +586,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -761,10 +749,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -810,10 +796,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -889,10 +873,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out index 37d97d2252..3ed461d825 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out @@ -158,10 +158,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -250,10 +248,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -298,10 +294,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -448,10 +442,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -540,10 +532,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -588,10 +578,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -734,10 +722,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -822,10 +808,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -870,10 +854,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1016,10 +998,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1107,10 +1087,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1155,10 +1133,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1229,10 +1205,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1277,10 +1251,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out index 6ef1f3433f..ce3fa3b6a4 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out @@ -196,10 +196,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -274,10 +272,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -381,10 +377,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -430,10 +424,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -501,10 +493,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out index f358a594b4..a6d73097e0 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_13.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -178,7 +179,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -366,7 +368,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -379,7 +382,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -567,7 +571,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -580,7 +585,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out index 90d362e981..eec2608616 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out @@ -157,10 +157,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -206,10 +204,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -285,10 +281,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -450,10 +444,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -499,10 +491,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -578,10 +568,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out index 365f63c0ad..536172c066 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out @@ -142,10 +142,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -191,10 +189,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -285,10 +281,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -450,10 +444,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -528,10 +520,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -577,10 +567,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -743,10 +731,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -821,10 +807,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -870,10 +854,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out index 8ee44b3493..8ab0210e80 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out @@ -158,10 +158,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -207,10 +205,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -301,10 +297,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -466,10 +460,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -544,10 +536,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -593,10 +583,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -759,10 +747,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -837,10 +823,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -886,10 +870,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out index 68b69e8ccd..0189ffdd83 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out @@ -82,16 +82,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -110,8 +110,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -132,8 +130,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -147,16 +143,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 4968 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4784 Basic stats: COMPLETE Column stats: NONE Merge Join Operator condition map: Inner Join 0 to 1 @@ -164,7 +160,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 5262 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -196,8 +192,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -218,8 +212,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -299,16 +291,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -327,8 +319,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -349,8 +339,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -364,16 +352,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 4968 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4784 Basic stats: COMPLETE Column stats: NONE Merge Join Operator condition map: Inner Join 0 to 1 @@ -381,7 +369,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 5262 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -413,8 +401,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -435,8 +421,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -517,27 +501,27 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 4968 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4784 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4784 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 3 => 1 + Estimated key counts: Map 3 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) input vertices: 1 Map 3 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 202 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 5262 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash @@ -571,8 +555,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -593,8 +575,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -609,22 +589,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: NONE tag: 1 auto parallelism: true Execution mode: llap @@ -648,8 +628,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -670,8 +648,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out index 83d5a968b7..b820827445 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out @@ -175,10 +175,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -224,10 +222,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -318,10 +314,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -367,10 +361,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -535,10 +527,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -584,10 +574,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -663,10 +651,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -712,10 +698,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -880,10 +864,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -929,10 +911,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1008,10 +988,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1057,10 +1035,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out index 0e0428481b..8698bc2ad6 100644 --- ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out +++ ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out @@ -175,10 +175,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -224,10 +222,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -318,10 +314,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -367,10 +361,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -535,10 +527,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -584,10 +574,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -663,10 +651,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -712,10 +698,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -880,10 +864,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -929,10 +911,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1008,10 +988,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1057,10 +1035,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/bucket2.q.out ql/src/test/results/clientpositive/llap/bucket2.q.out index e0c92ced9f..9954c2dd99 100644 --- ql/src/test/results/clientpositive/llap/bucket2.q.out +++ ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -174,7 +174,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/llap/bucket3.q.out ql/src/test/results/clientpositive/llap/bucket3.q.out index 8e6d85ca80..218f9b7e48 100644 --- ql/src/test/results/clientpositive/llap/bucket3.q.out +++ ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -171,7 +171,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') diff --git ql/src/test/results/clientpositive/llap/bucket4.q.out ql/src/test/results/clientpositive/llap/bucket4.q.out index 5fbffc96a9..2115565096 100644 --- ql/src/test/results/clientpositive/llap/bucket4.q.out +++ ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -177,7 +177,8 @@ STAGE PLANS: name: default.bucket4_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket4_1 diff --git ql/src/test/results/clientpositive/llap/bucket5.q.out ql/src/test/results/clientpositive/llap/bucket5.q.out index 23576234c6..680dbd95d2 100644 --- ql/src/test/results/clientpositive/llap/bucket5.q.out +++ ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -248,7 +248,8 @@ STAGE PLANS: name: default.bucketed_table Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -280,7 +281,8 @@ STAGE PLANS: name: default.unbucketed_table Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 diff --git ql/src/test/results/clientpositive/llap/bucket6.q.out ql/src/test/results/clientpositive/llap/bucket6.q.out index 20895f8a9f..42f062b0f6 100644 --- ql/src/test/results/clientpositive/llap/bucket6.q.out +++ ql/src/test/results/clientpositive/llap/bucket6.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.src_bucket Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table src_bucket select key,value from srcpart PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/bucket_many.q.out ql/src/test/results/clientpositive/llap/bucket_many.q.out index b78cbaa0c3..4f3bee20ce 100644 --- ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -174,7 +174,8 @@ STAGE PLANS: name: default.bucket_many Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket_many diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out index c6ab95ca53..fca752692c 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out @@ -406,22 +406,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: true @@ -445,8 +445,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -466,8 +464,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -523,10 +519,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -638,7 +632,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -772,22 +767,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: true @@ -811,8 +806,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -832,8 +825,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -889,10 +880,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1004,7 +993,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out index c254d89dfa..3dd0bf90f7 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out @@ -159,10 +159,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -237,10 +235,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -352,7 +348,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -531,10 +528,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -609,10 +604,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -724,7 +717,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -920,10 +914,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -998,10 +990,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1046,10 +1036,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1162,7 +1150,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out index 167813553f..dd9d1ab9b3 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out @@ -183,10 +183,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -261,10 +259,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -376,7 +372,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -555,10 +552,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -633,10 +628,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -748,7 +741,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out index b871c9365b..fcf181192c 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out @@ -142,22 +142,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: true @@ -181,8 +181,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -202,8 +200,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -218,22 +214,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string) auto parallelism: true @@ -257,8 +253,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -278,8 +272,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -302,17 +294,17 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToString(_col0) (type: string), _col1 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -372,7 +364,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -498,22 +491,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: true @@ -537,8 +530,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -558,8 +549,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -574,22 +563,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string) auto parallelism: true @@ -613,8 +602,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -634,8 +621,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -658,17 +643,17 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToString(_col0) (type: string), _col1 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 206 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 5170 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -728,7 +713,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out index b3ffda2679..867daf543b 100644 --- ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out +++ ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out @@ -118,10 +118,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -193,10 +191,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out index 13eecea578..b907c2dbd8 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out @@ -189,7 +189,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -368,7 +369,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -571,7 +573,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -780,7 +783,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -971,7 +975,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.v1, b.v2) @@ -1162,7 +1167,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key+a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out index 9959ba4738..f5f5f91e82 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_6.q.out @@ -167,7 +167,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key2, concat(a.value, b.value) @@ -353,7 +354,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq1.key, subq1.key2, subq1.value from @@ -539,7 +541,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -660,7 +663,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -787,7 +791,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.key, subq2.key2, subq2.value from @@ -991,7 +996,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.k2, subq2.k1, subq2.value from @@ -1205,5 +1211,6 @@ STAGE PLANS: name: default.test_table4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out index c5e2973701..7b380562ac 100644 --- ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out @@ -167,7 +167,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -350,7 +351,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -539,7 +541,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out index c60a5beb32..c3d1907c76 100644 --- ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out +++ ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -958,8 +958,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/column_table_stats.q.out ql/src/test/results/clientpositive/llap/column_table_stats.q.out index f073720fac..b1f83434ff 100644 --- ql/src/test/results/clientpositive/llap/column_table_stats.q.out +++ ql/src/test/results/clientpositive/llap/column_table_stats.q.out @@ -36,8 +36,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -58,7 +56,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -72,13 +69,13 @@ STAGE PLANS: Map Operator Tree: TableScan alias: s - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 10672 Basic stats: COMPLETE Column stats: NONE Statistics Aggregation Key Prefix: default.s/ GatherStats: true Select Operator expressions: key (type: string), value (type: string) outputColumnNames: key, value - Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 10672 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') mode: hash @@ -110,8 +107,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.s numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct s { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -130,8 +125,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.s numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct s { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -175,11 +168,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.s/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.s/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -314,7 +305,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -372,10 +362,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.spart numFiles 1 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct spart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -419,10 +407,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.spart numFiles 1 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct spart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -490,11 +476,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.spart/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.spart/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -723,7 +707,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -781,10 +764,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.spart numFiles 1 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct spart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -828,10 +809,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.spart numFiles 1 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct spart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -899,11 +878,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.spart/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.spart/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -1132,7 +1109,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -1190,10 +1166,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.spart numFiles 1 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct spart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1260,11 +1234,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.spart/ - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.spart/ Column Stats Desc: Columns: key, value Column Types: string, string @@ -1384,8 +1356,6 @@ Table: spart #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 5812 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out index 4bc5b3933e..6e71803655 100644 --- ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out +++ ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out @@ -59,7 +59,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -178,10 +177,8 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: key, value Column Types: string, string @@ -311,7 +308,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -481,10 +477,8 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: key, value Column Types: string, string @@ -708,7 +702,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -833,10 +826,8 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: key, value Column Types: string, string diff --git ql/src/test/results/clientpositive/llap/ctas.q.out ql/src/test/results/clientpositive/llap/ctas.q.out index 1dd2dd19e4..7240da636b 100644 --- ql/src/test/results/clientpositive/llap/ctas.q.out +++ ql/src/test/results/clientpositive/llap/ctas.q.out @@ -97,7 +97,8 @@ STAGE PLANS: name: default.nzhang_CTAS1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -252,7 +253,8 @@ STAGE PLANS: name: default.nzhang_ctas2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -407,7 +409,8 @@ STAGE PLANS: name: default.nzhang_ctas3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -626,7 +629,8 @@ STAGE PLANS: name: default.nzhang_ctas4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -785,7 +789,8 @@ STAGE PLANS: name: default.nzhang_ctas5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index 45b9af6760..d2552fe867 100644 --- ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -94,6 +94,42 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 POSTHOOK: Output: default@testdeci2 #### A masked pattern was here #### +PREHOOK: query: describe formatted testdeci2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testdeci2 +POSTHOOK: query: describe formatted testdeci2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testdeci2 +# col_name data_type comment +id int +amount decimal(10,3) +sales_tax decimal(10,3) +item string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"amount\":\"true\",\"id\":\"true\",\"item\":\"true\",\"sales_tax\":\"true\"}} + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 @@ -129,18 +165,18 @@ Stage-0 Stage-1 Reducer 2 llap File Output Operator [FS_10] - Select Operator [SEL_9] (rows=2 width=228) + Select Operator [SEL_9] (rows=1 width=228) Output:["_col0","_col1","_col2"] - Merge Join Operator [MERGEJOIN_15] (rows=2 width=228) + Merge Join Operator [MERGEJOIN_15] (rows=1 width=228) Conds:RS_6._col1=RS_7._col3(Inner),Output:["_col0","_col3","_col4"] <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_6] PartitionCols:_col1 - Select Operator [SEL_2] (rows=2 width=88) + Select Operator [SEL_2] (rows=1 width=88) Output:["_col0","_col1"] - Filter Operator [FIL_13] (rows=2 width=88) + Filter Operator [FIL_13] (rows=1 width=88) predicate:item is not null - TableScan [TS_0] (rows=2 width=88) + TableScan [TS_0] (rows=1 width=88) default@testdeci2,s,Tbl:COMPLETE,Col:COMPLETE,Output:["id","item"] <-Map 3 [SIMPLE_EDGE] llap SHUFFLE [RS_7] @@ -149,6 +185,6 @@ Stage-0 Output:["_col1","_col2","_col3"] Filter Operator [FIL_14] (rows=1 width=312) predicate:((id = 2) and item is not null) - TableScan [TS_3] (rows=2 width=312) + TableScan [TS_3] (rows=1 width=312) default@testdeci2,d,Tbl:COMPLETE,Col:COMPLETE,Output:["id","amount","sales_tax","item"] diff --git ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index e31ebaf5d3..acd607628a 100644 --- ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -174,7 +174,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index de867b26b3..8ad8e25d40 100644 --- ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out index 374d4b84de..6772e5dd58 100644 --- ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out +++ ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out @@ -259,7 +259,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -273,7 +274,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -286,7 +288,8 @@ STAGE PLANS: name: default.merge_tmp_table Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -300,7 +303,8 @@ STAGE PLANS: Write Type: INSERT Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) @@ -433,7 +437,8 @@ STAGE PLANS: Write Type: INSERT Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain merge into acidTbl as t using ( select * from nonAcidOrcTbl where a > 0 @@ -734,7 +739,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -748,7 +754,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -761,7 +768,8 @@ STAGE PLANS: name: default.merge_tmp_table Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -775,7 +783,8 @@ STAGE PLANS: Write Type: INSERT Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop database if exists type2_scd_helper cascade PREHOOK: type: DROPDATABASE diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index 2e62fb08f3..ecc467a08e 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -223,7 +223,8 @@ STAGE PLANS: name: default.over1k_part_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_limit_orc partition(ds="foo", t) select si,i,b,f,t from over1k_orc where t is null or t=27 limit 10 PREHOOK: type: QUERY @@ -316,7 +317,8 @@ STAGE PLANS: name: default.over1k_part_limit_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_buck_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27 PREHOOK: type: QUERY @@ -389,7 +391,8 @@ STAGE PLANS: name: default.over1k_part_buck_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_buck_sort_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27 PREHOOK: type: QUERY @@ -462,7 +465,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part_orc partition(ds="foo", t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by si PREHOOK: type: QUERY @@ -604,7 +608,8 @@ STAGE PLANS: name: default.over1k_part_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table over1k_part_limit_orc partition(ds="foo", t) select si,i,b,f,t from over1k_orc where t is null or t=27 limit 10 PREHOOK: type: QUERY @@ -697,7 +702,8 @@ STAGE PLANS: name: default.over1k_part_limit_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table over1k_part_buck_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27 PREHOOK: type: QUERY @@ -770,7 +776,8 @@ STAGE PLANS: name: default.over1k_part_buck_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table over1k_part_buck_sort_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27 PREHOOK: type: QUERY @@ -843,7 +850,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table over1k_part_orc partition(ds="foo", t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by si PREHOOK: type: QUERY @@ -1369,7 +1377,8 @@ STAGE PLANS: name: default.over1k_part2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i PREHOOK: type: QUERY @@ -1443,7 +1452,8 @@ STAGE PLANS: name: default.over1k_part2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_orc order by i limit 10) tmp where t is null or t=27 PREHOOK: type: QUERY @@ -1538,7 +1548,8 @@ STAGE PLANS: name: default.over1k_part2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 group by si,i,b,f,t PREHOOK: type: QUERY @@ -1616,7 +1627,8 @@ STAGE PLANS: name: default.over1k_part2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 group by si,i,b,f,t PREHOOK: type: QUERY @@ -1695,7 +1707,8 @@ STAGE PLANS: name: default.over1k_part2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i PREHOOK: type: QUERY @@ -2071,7 +2084,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27 PREHOOK: type: QUERY @@ -2144,7 +2158,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort2_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index 5b80a320f2..764b58e9c7 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -180,7 +180,8 @@ STAGE PLANS: name: default.over1k_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k where t is null or t=27 limit 10 PREHOOK: type: QUERY @@ -273,7 +274,8 @@ STAGE PLANS: name: default.over1k_part_limit Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -346,7 +348,8 @@ STAGE PLANS: name: default.over1k_part_buck Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -419,7 +422,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -561,7 +565,8 @@ STAGE PLANS: name: default.over1k_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k where t is null or t=27 limit 10 PREHOOK: type: QUERY @@ -654,7 +659,8 @@ STAGE PLANS: name: default.over1k_part_limit Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -727,7 +733,8 @@ STAGE PLANS: name: default.over1k_part_buck Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -800,7 +807,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -1326,7 +1334,8 @@ STAGE PLANS: name: default.over1k_part2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k where t is null or t=27 order by i PREHOOK: type: QUERY @@ -1400,7 +1409,8 @@ STAGE PLANS: name: default.over1k_part2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k order by i limit 10) tmp where t is null or t=27 PREHOOK: type: QUERY @@ -1495,7 +1505,8 @@ STAGE PLANS: name: default.over1k_part2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k where t is null or t=27 group by si,i,b,f,t PREHOOK: type: QUERY @@ -1573,7 +1584,8 @@ STAGE PLANS: name: default.over1k_part2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k where t is null or t=27 group by si,i,b,f,t PREHOOK: type: QUERY @@ -1652,7 +1664,8 @@ STAGE PLANS: name: default.over1k_part2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k where t is null or t=27 order by i PREHOOK: type: QUERY @@ -2028,7 +2041,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -2101,7 +2115,8 @@ STAGE PLANS: name: default.over1k_part_buck_sort2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k where t is null or t=27 PREHOOK: type: QUERY @@ -2470,7 +2485,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 PREHOOK: type: QUERY @@ -2545,7 +2561,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 PREHOOK: type: QUERY @@ -2620,7 +2637,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 PREHOOK: type: QUERY @@ -2695,7 +2713,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo" PREHOOK: type: QUERY @@ -2770,7 +2789,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo" PREHOOK: type: QUERY @@ -2845,7 +2865,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo" PREHOOK: type: QUERY @@ -2903,7 +2924,8 @@ STAGE PLANS: name: default.over1k_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo" PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index 568ab65007..8b0fb2c44f 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -144,7 +144,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -378,7 +379,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -616,7 +618,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -848,7 +851,8 @@ STAGE PLANS: name: default.ss_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -1141,7 +1145,8 @@ STAGE PLANS: name: default.ss_part_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part_orc partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -1373,7 +1378,8 @@ STAGE PLANS: name: default.ss_part_orc Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table ss_part_orc partition (ss_sold_date_sk) select ss_net_paid_inc_tax, @@ -1647,7 +1653,8 @@ STAGE PLANS: name: default.hive13_dp1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table `hive13_dp1` partition(`day`) select @@ -1777,7 +1784,8 @@ STAGE PLANS: name: default.hive13_dp1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table `hive13_dp1` partition(`day`) select diff --git ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 16c0b29be7..02cadb7cff 100644 --- ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -141,7 +141,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part set value = 'bar' where key = 'foo' and ds='2008-04-08' PREHOOK: type: QUERY @@ -236,7 +237,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part set value = 'bar' where key = 'foo' and ds in ('2008-04-08') PREHOOK: type: QUERY @@ -423,7 +425,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' PREHOOK: type: QUERY @@ -518,7 +521,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_part_sdpo set value = 'bar' where key = 'foo' and ds in ('2008-04-08') PREHOOK: type: QUERY @@ -715,7 +719,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 PREHOOK: type: QUERY @@ -811,7 +816,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 PREHOOK: type: QUERY @@ -936,7 +942,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from acid_2L_part where value = 'bar' PREHOOK: type: QUERY @@ -1130,7 +1137,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 PREHOOK: type: QUERY @@ -1226,7 +1234,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 PREHOOK: type: QUERY @@ -1351,7 +1360,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: delete from acid_2L_part_sdpo where value = 'bar' PREHOOK: type: QUERY @@ -1547,7 +1557,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 PREHOOK: type: QUERY @@ -1644,7 +1655,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: update acid_2L_part_sdpo_no_cp set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/explainuser_1.q.out ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 4452dea71f..3bd7d3b777 100644 --- ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -45,7 +45,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -80,7 +80,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -3177,7 +3177,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.nzhang_CTAS1 @@ -3230,7 +3230,7 @@ Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.nzhang_ctas3 @@ -4832,7 +4832,7 @@ Reducer 4 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.part_4"} @@ -4883,7 +4883,7 @@ Stage-4 PartitionCols:_col2 Please refer to the previous PTF Operator [PTF_3] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.part_5"} @@ -5248,7 +5248,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest_j1"} diff --git ql/src/test/results/clientpositive/llap/explainuser_2.q.out ql/src/test/results/clientpositive/llap/explainuser_2.q.out index e5ba529d05..6b678f3d4c 100644 --- ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -2607,7 +2607,7 @@ Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 11 (SIMPLE_EDGE) Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE), Union 4 (CONTAINS) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -2840,13 +2840,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_44] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -2910,7 +2910,7 @@ Reducer 6 <- Union 5 (SIMPLE_EDGE), Union 7 (CONTAINS) Reducer 8 <- Union 7 (SIMPLE_EDGE) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -3172,13 +3172,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_120] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -3223,7 +3223,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -3278,7 +3278,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_13] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -3397,7 +3397,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -3470,7 +3470,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -3505,7 +3505,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -3564,7 +3564,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} diff --git ql/src/test/results/clientpositive/llap/groupby1.q.out ql/src/test/results/clientpositive/llap/groupby1.q.out index 5917013fa4..d58a9fd69f 100644 --- ql/src/test/results/clientpositive/llap/groupby1.q.out +++ ql/src/test/results/clientpositive/llap/groupby1.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/groupby2.q.out ql/src/test/results/clientpositive/llap/groupby2.q.out index f94db4edf1..5ab62358ed 100644 --- ql/src/test/results/clientpositive/llap/groupby2.q.out +++ ql/src/test/results/clientpositive/llap/groupby2.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/llap/groupby3.q.out ql/src/test/results/clientpositive/llap/groupby3.q.out index 3495de6d3f..897946b4ba 100644 --- ql/src/test/results/clientpositive/llap/groupby3.q.out +++ ql/src/test/results/clientpositive/llap/groupby3.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/llap/insert1.q.out ql/src/test/results/clientpositive/llap/insert1.q.out index 0e27f9dfab..bc626824f0 100644 --- ql/src/test/results/clientpositive/llap/insert1.q.out +++ ql/src/test/results/clientpositive/llap/insert1.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY @@ -130,7 +131,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create database x PREHOOK: type: CREATEDATABASE @@ -198,7 +200,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table default.INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY @@ -252,7 +255,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain from insert2 @@ -329,7 +333,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -342,7 +347,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE DATABASE db2 PREHOOK: type: CREATEDATABASE diff --git ql/src/test/results/clientpositive/llap/insert_into1.q.out ql/src/test/results/clientpositive/llap/insert_into1.q.out index df72ca120d..61297f04c8 100644 --- ql/src/test/results/clientpositive/llap/insert_into1.q.out +++ ql/src/test/results/clientpositive/llap/insert_into1.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -204,7 +205,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -326,7 +328,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 10 PREHOOK: type: QUERY @@ -426,7 +429,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table insert_into1 select 1, 'a' PREHOOK: type: QUERY @@ -488,7 +492,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into insert_into1 select 2, 'b' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/insert_into2.q.out ql/src/test/results/clientpositive/llap/insert_into2.q.out index a42c651d63..ef2b875efb 100644 --- ql/src/test/results/clientpositive/llap/insert_into2.q.out +++ ql/src/test/results/clientpositive/llap/insert_into2.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into2 PARTITION (ds='1') SELECT * FROM src order by key limit 100 PREHOOK: type: QUERY @@ -251,7 +252,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 100 @@ -383,7 +385,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 50 diff --git ql/src/test/results/clientpositive/llap/join1.q.out ql/src/test/results/clientpositive/llap/join1.q.out index 724481beef..661f55caaf 100644 --- ql/src/test/results/clientpositive/llap/join1.q.out +++ ql/src/test/results/clientpositive/llap/join1.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/llap/join32_lessSize.q.out ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index 2efe092165..140f87ea42 100644 --- ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -364,7 +364,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -932,7 +933,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -1403,7 +1405,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -1881,7 +1884,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -2150,7 +2154,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, x.value, res.value @@ -2406,7 +2411,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, y.value, res.value diff --git ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index 23c28e0277..edbf76d935 100644 --- ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/llap/llap_stats.q.out ql/src/test/results/clientpositive/llap/llap_stats.q.out index 4a48627597..299d9b4dbb 100644 --- ql/src/test/results/clientpositive/llap/llap_stats.q.out +++ ql/src/test/results/clientpositive/llap/llap_stats.q.out @@ -94,7 +94,6 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 - Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -149,10 +148,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 - Stats-Aggr Operator - - Stage: Stage-3 - Column Stats Work + Stats Work + Basic Stats NoJob Work: Column Stats Desc: Columns: ctinyint, csmallint Column Types: tinyint, smallint diff --git ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out index 41f93cc3b0..2b89d24027 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part1.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -135,7 +136,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out index f15226b3d0..52b73536dd 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part2.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.nzhang_part_bucket Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part_bucket partition (ds='2010-03-23', hr) select key, value, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out index fd24a05274..5230b597c0 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part3.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.nzhang_part3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part3 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index 8e2bf0c709..8b98b8ed9c 100644 --- ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.nzhang_part5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part5 partition (value) select key, value from src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out index ef26e92e8a..134ade7053 100644 --- ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out +++ ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out @@ -456,7 +456,7 @@ STAGE PLANS: TableScan alias: part_null filterExpr: p_name is null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 5888 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: p_name is null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE @@ -523,7 +523,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Map 1' is a cross product +Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Reducer 3' is a cross product PREHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) @@ -537,8 +537,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Reducer 3 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (CUSTOM_SIMPLE_EDGE), Map 4 (BROADCAST_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -554,30 +553,10 @@ STAGE PLANS: expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 - 1 - 2 - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - input vertices: - 1 Reducer 3 - 2 Map 2 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs Map 2 @@ -585,7 +564,7 @@ STAGE PLANS: TableScan alias: part_null filterExpr: p_name is null (type: boolean) - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 32 Data size: 5888 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: p_name is null (type: boolean) Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE @@ -600,6 +579,19 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: part_null + filterExpr: p_name is null (type: boolean) + Statistics: Num rows: 32 Data size: 5888 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: p_name is null (type: boolean) + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE @@ -618,9 +610,30 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 192 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 + 1 + 2 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + input vertices: + 0 Map 1 + 2 Map 4 + Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/llap/mapreduce1.q.out ql/src/test/results/clientpositive/llap/mapreduce1.q.out index 37f92d9f20..0e94e71d27 100644 --- ql/src/test/results/clientpositive/llap/mapreduce1.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce1.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/llap/mapreduce2.q.out ql/src/test/results/clientpositive/llap/mapreduce2.q.out index 71bbb7e612..6485f587f8 100644 --- ql/src/test/results/clientpositive/llap/mapreduce2.q.out +++ ql/src/test/results/clientpositive/llap/mapreduce2.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/llap/merge1.q.out ql/src/test/results/clientpositive/llap/merge1.q.out index ec794dc1ea..e77a5dfe21 100644 --- ql/src/test/results/clientpositive/llap/merge1.q.out +++ ql/src/test/results/clientpositive/llap/merge1.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest1 select key, count(1) from src group by key @@ -513,7 +514,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest1 select key from test_src PREHOOK: type: QUERY @@ -579,7 +581,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table dest1 select key from test_src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/merge2.q.out ql/src/test/results/clientpositive/llap/merge2.q.out index 6ad6864eb3..25b194697a 100644 --- ql/src/test/results/clientpositive/llap/merge2.q.out +++ ql/src/test/results/clientpositive/llap/merge2.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table test1 select key, count(1) from src group by key @@ -513,7 +514,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table test1 select key from test_src PREHOOK: type: QUERY @@ -579,7 +581,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table test1 select key from test_src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/multi_insert.q.out ql/src/test/results/clientpositive/llap/multi_insert.q.out index 56d26b30b4..7f30afeef3 100644 --- ql/src/test/results/clientpositive/llap/multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -102,7 +103,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -232,7 +234,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -245,7 +248,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -375,7 +379,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -388,7 +393,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -518,7 +524,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -531,7 +538,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -679,7 +687,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -692,7 +701,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -833,7 +843,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -846,7 +857,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -987,7 +999,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1000,7 +1013,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1141,7 +1155,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1154,7 +1169,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1312,7 +1328,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1325,7 +1342,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1509,7 +1527,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1522,7 +1541,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1706,7 +1726,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1719,7 +1740,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1903,7 +1925,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1916,7 +1939,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 diff --git ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out index 6d20939009..df92c0004f 100644 --- ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out +++ ql/src/test/results/clientpositive/llap/multi_insert_lateral_view.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -178,7 +179,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C @@ -431,7 +433,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -444,7 +447,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -673,7 +677,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -686,7 +691,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -699,7 +705,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -983,7 +990,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -996,7 +1004,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1009,7 +1018,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C @@ -1354,7 +1364,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1367,7 +1378,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1380,7 +1392,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1393,7 +1406,8 @@ STAGE PLANS: name: default.src_lv4 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key diff --git ql/src/test/results/clientpositive/llap/orc_merge1.q.out ql/src/test/results/clientpositive/llap/orc_merge1.q.out index ba29491001..5669a5baf4 100644 --- ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -191,7 +192,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -320,7 +322,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez diff --git ql/src/test/results/clientpositive/llap/orc_merge10.q.out ql/src/test/results/clientpositive/llap/orc_merge10.q.out index dd5d1cbbe7..4671e477e0 100644 --- ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -191,7 +192,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -320,7 +322,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -459,7 +462,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge2.q.out ql/src/test/results/clientpositive/llap/orc_merge2.q.out index c38852a95b..53a64248a3 100644 --- ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, diff --git ql/src/test/results/clientpositive/llap/orc_merge3.q.out ql/src/test/results/clientpositive/llap/orc_merge3.q.out index c3ca701152..dca37ecd2b 100644 --- ql/src/test/results/clientpositive/llap/orc_merge3.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge3.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b SELECT key, value FROM orcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/orc_merge4.q.out ql/src/test/results/clientpositive/llap/orc_merge4.q.out index a16d2bf11b..a2aa416681 100644 --- ql/src/test/results/clientpositive/llap/orc_merge4.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge4.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3b SELECT key, value FROM orcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/orc_merge5.q.out ql/src/test/results/clientpositive/llap/orc_merge5.q.out index 8de6935aac..fafba53f84 100644 --- ql/src/test/results/clientpositive/llap/orc_merge5.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge5.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -178,7 +179,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -299,7 +301,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5b concatenate PREHOOK: type: ALTER_TABLE_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge6.q.out ql/src/test/results/clientpositive/llap/orc_merge6.q.out index b71e1496c4..e5672fd40a 100644 --- ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (year="2000",hour=24) select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -226,7 +227,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -434,7 +436,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(year="2000",hour=24) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge7.q.out ql/src/test/results/clientpositive/llap/orc_merge7.q.out index 1e644df919..f224822ec9 100644 --- ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 PREHOOK: type: QUERY @@ -255,7 +256,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -540,7 +542,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out index ba29491001..5669a5baf4 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -191,7 +192,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez @@ -320,7 +322,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out index b6c1201fba..ae7250a6c0 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat1.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index a0472cef13..c1822a17fc 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 order by userid PREHOOK: type: QUERY @@ -286,7 +287,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/llap/parallel.q.out ql/src/test/results/clientpositive/llap/parallel.q.out index aea94170ea..3beb3400d7 100644 --- ql/src/test/results/clientpositive/llap/parallel.q.out +++ ql/src/test/results/clientpositive/llap/parallel.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -134,7 +135,8 @@ STAGE PLANS: name: default.src_b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/llap/parallel_colstats.q.out ql/src/test/results/clientpositive/llap/parallel_colstats.q.out index e89bf2f177..4ac3fbb2f5 100644 --- ql/src/test/results/clientpositive/llap/parallel_colstats.q.out +++ ql/src/test/results/clientpositive/llap/parallel_colstats.q.out @@ -29,8 +29,6 @@ STAGE DEPENDENCIES: Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 Stage-4 depends on stages: Stage-0 - Stage-6 depends on stages: Stage-4, Stage-5 - Stage-7 depends on stages: Stage-4, Stage-5 Stage-1 depends on stages: Stage-3 Stage-5 depends on stages: Stage-1 @@ -181,22 +179,13 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-6 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string Table: default.src_a - Stage: Stage-7 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.src_b - Stage: Stage-1 Move Operator tables: @@ -208,7 +197,12 @@ STAGE PLANS: name: default.src_b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.src_b PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/llap/ptf.q.out ql/src/test/results/clientpositive/llap/ptf.q.out index c69ed1ddc0..4f79e260ae 100644 --- ql/src/test/results/clientpositive/llap/ptf.q.out +++ ql/src/test/results/clientpositive/llap/ptf.q.out @@ -3181,7 +3181,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3194,7 +3195,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part partition by p_mfgr diff --git ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out index 42d3bd0a14..066295f090 100644 --- ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_createas1.q.out @@ -97,7 +97,8 @@ STAGE PLANS: name: default.rcfile_createas1b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out index 2bb8e8ab1d..e2ecc76948 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.rcfile_merge2a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge2a PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, diff --git ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out index d027bd3ec5..02ee0e0181 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge3.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.rcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge3b SELECT key, value FROM rcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out index f4a0353348..52c97d11d5 100644 --- ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out +++ ql/src/test/results/clientpositive/llap/rcfile_merge4.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.rcfile_merge3b Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE rcfile_merge3b SELECT key, value FROM rcfile_merge3a diff --git ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 40524f1515..65b74ee319 100644 --- ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -175,7 +175,8 @@ STAGE PLANS: name: default.bucket5_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket5_1 @@ -388,6 +389,7 @@ STAGE PLANS: name: default.complex_tbl_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/llap/sample1.q.out ql/src/test/results/clientpositive/llap/sample1.q.out index 1a7fb3254e..5b69bb5832 100644 --- ql/src/test/results/clientpositive/llap/sample1.q.out +++ ql/src/test/results/clientpositive/llap/sample1.q.out @@ -161,7 +161,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/llap/skewjoin.q.out ql/src/test/results/clientpositive/llap/skewjoin.q.out index e53da811de..b102bca4fe 100644 --- ql/src/test/results/clientpositive/llap/skewjoin.q.out +++ ql/src/test/results/clientpositive/llap/skewjoin.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out index c061df6f72..135f83eca9 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_18.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' @@ -281,7 +282,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' and a.key = 238 @@ -405,7 +407,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table2 a WHERE a.ds = '2' diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out index 1f4fcae83c..64763fae40 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' diff --git ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out index 91b8f81cb6..07e669fff7 100644 --- ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/llap/smb_mapjoin_6.q.out @@ -150,7 +150,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -1332,7 +1333,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -2530,7 +2532,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 @@ -2644,7 +2647,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 diff --git ql/src/test/results/clientpositive/llap/sqlmerge.q.out ql/src/test/results/clientpositive/llap/sqlmerge.q.out index 5fc35d5928..2a6a0f32de 100644 --- ql/src/test/results/clientpositive/llap/sqlmerge.q.out +++ ql/src/test/results/clientpositive/llap/sqlmerge.q.out @@ -229,7 +229,8 @@ STAGE PLANS: Write Type: DELETE Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -243,7 +244,8 @@ STAGE PLANS: Write Type: UPDATE Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -256,7 +258,8 @@ STAGE PLANS: name: default.merge_tmp_table Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -270,7 +273,8 @@ STAGE PLANS: Write Type: INSERT Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain merge into acidTbl as t using nonAcidOrcTbl s ON t.a = s.a WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b) @@ -373,5 +377,6 @@ STAGE PLANS: Write Type: INSERT Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/stats11.q.out ql/src/test/results/clientpositive/llap/stats11.q.out index 2899d8a033..92841d9734 100644 --- ql/src/test/results/clientpositive/llap/stats11.q.out +++ ql/src/test/results/clientpositive/llap/stats11.q.out @@ -54,7 +54,8 @@ STAGE PLANS: name: default.srcbucket_mapjoin_part Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') PREHOOK: type: LOAD @@ -86,8 +87,6 @@ Table: srcbucket_mapjoin_part #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 1358 #### A masked pattern was here #### @@ -130,8 +129,6 @@ Table: srcbucket_mapjoin_part #### A masked pattern was here #### Partition Parameters: numFiles 2 - numRows 0 - rawDataSize 0 totalSize 2750 #### A masked pattern was here #### @@ -174,8 +171,6 @@ Table: srcbucket_mapjoin_part #### A masked pattern was here #### Partition Parameters: numFiles 3 - numRows 0 - rawDataSize 0 totalSize 4200 #### A masked pattern was here #### @@ -218,8 +213,6 @@ Table: srcbucket_mapjoin_part #### A masked pattern was here #### Partition Parameters: numFiles 4 - numRows 0 - rawDataSize 0 totalSize 5812 #### A masked pattern was here #### @@ -312,22 +305,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: true @@ -351,8 +344,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -372,8 +363,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -429,10 +418,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -544,7 +531,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -678,22 +666,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 4888 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 25 Data size: 4700 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: true @@ -717,8 +705,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -738,8 +724,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -795,10 +779,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -910,7 +892,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out index 6d3e6740a3..306f0de48c 100644 --- ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out +++ ql/src/test/results/clientpositive/llap/stats_noscan_1.q.out @@ -44,7 +44,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan PREHOOK: type: QUERY @@ -305,7 +306,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/temp_table.q.out ql/src/test/results/clientpositive/llap/temp_table.q.out index 462edae4fe..4cf47b5bd9 100644 --- ql/src/test/results/clientpositive/llap/temp_table.q.out +++ ql/src/test/results/clientpositive/llap/temp_table.q.out @@ -52,7 +52,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -124,7 +125,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/tez_dml.q.out ql/src/test/results/clientpositive/llap/tez_dml.q.out index 84c65a7e66..adccaae2aa 100644 --- ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.tmp_src Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -480,7 +481,8 @@ STAGE PLANS: name: default.tmp_src_part Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE tmp_src_part PARTITION (d) SELECT * FROM tmp_src PREHOOK: type: QUERY @@ -917,7 +919,8 @@ STAGE PLANS: name: default.even Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -930,7 +933,8 @@ STAGE PLANS: name: default.odd Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT INTO TABLE even SELECT key, value WHERE key % 2 = 0 diff --git ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out index 417d318367..487c0f2fcc 100644 --- ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out +++ ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out @@ -167,22 +167,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: contact_event_id is not null (type: boolean) - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: contact_event_id (type: string), ce_create_dt (type: string), ce_end_dt (type: string), contact_type (type: string), cnctevs_cd (type: string), contact_mode (type: string), cntvnst_stts_cd (type: string), total_transfers (type: int), ce_notes (type: array) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: array) auto parallelism: true @@ -209,8 +209,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.ct_events_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct ct_events_clean { string contact_event_id, string ce_create_dt, string ce_end_dt, string contact_type, string cnctevs_cd, string contact_mode, string cntvnst_stts_cd, i32 total_transfers, list ce_notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -233,8 +231,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.ct_events_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct ct_events_clean { string contact_event_id, string ce_create_dt, string ce_end_dt, string contact_type, string cnctevs_cd, string contact_mode, string cntvnst_stts_cd, i32 total_transfers, list ce_notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -249,20 +245,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 4944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 19776 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: cnctevn_id is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 19776 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cnctevn_id (type: string), svcrqst_id (type: string), cnctmd_cd (type: string), svcrtyp_cd (type: string), cmpltyp_cd (type: string), sum_reason_cd (type: string), svcrqct_cds (type: array), notes (type: array) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 4944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 19776 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Map 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) @@ -270,17 +266,17 @@ STAGE PLANS: input vertices: 0 Map 1 Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 3533 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 21753 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: array), _col10 (type: string), _col15 (type: array), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col11 (type: string), _col16 (type: array) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 1 Data size: 3533 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 21753 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 3533 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 21753 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -319,8 +315,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.service_request_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct service_request_clean { string cnctevn_id, string svcrqst_id, string svcrqst_crt_dts, i32 subject_seq_no, string plan_component, string cust_segment, string cnctyp_cd, string cnctmd_cd, string cnctevs_cd, string svcrtyp_cd, string svrstyp_cd, string cmpltyp_cd, string catsrsn_cd, string apealvl_cd, string cnstnty_cd, string svcrqst_asrqst_ind, string svcrqst_rtnorig_in, string svcrqst_vwasof_dt, string sum_reason_cd, string sum_reason, string crsr_master_claim_index, list svcrqct_cds, string svcrqst_lupdt, timestamp crsr_lupdt, string cntevsds_lupdt, i32 ignore_me, list notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -343,8 +337,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.service_request_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct service_request_clean { string cnctevn_id, string svcrqst_id, string svcrqst_crt_dts, i32 subject_seq_no, string plan_component, string cust_segment, string cnctyp_cd, string cnctmd_cd, string cnctevs_cd, string svcrtyp_cd, string svrstyp_cd, string cmpltyp_cd, string catsrsn_cd, string apealvl_cd, string cnstnty_cd, string svcrqst_asrqst_ind, string svcrqst_rtnorig_in, string svcrqst_vwasof_dt, string sum_reason_cd, string sum_reason, string crsr_master_claim_index, list svcrqct_cds, string svcrqst_lupdt, timestamp crsr_lupdt, string cntevsds_lupdt, i32 ignore_me, list notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -369,7 +361,8 @@ STAGE PLANS: name: default.ct_events1_test Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-0 @@ -1151,22 +1144,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: contact_event_id is not null (type: boolean) - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: contact_event_id (type: string), ce_create_dt (type: string), ce_end_dt (type: string), contact_type (type: string), cnctevs_cd (type: string), contact_mode (type: string), cntvnst_stts_cd (type: string), total_transfers (type: int), ce_notes (type: array) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 3212 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6424 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: array) auto parallelism: true @@ -1193,8 +1186,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.ct_events_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct ct_events_clean { string contact_event_id, string ce_create_dt, string ce_end_dt, string contact_type, string cnctevs_cd, string contact_mode, string cntvnst_stts_cd, i32 total_transfers, list ce_notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1217,8 +1208,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.ct_events_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct ct_events_clean { string contact_event_id, string ce_create_dt, string ce_end_dt, string contact_type, string cnctevs_cd, string contact_mode, string cntvnst_stts_cd, i32 total_transfers, list ce_notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1233,20 +1222,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 4944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 19776 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: cnctevn_id is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 19776 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: cnctevn_id (type: string), svcrqst_id (type: string), cnctmd_cd (type: string), svcrtyp_cd (type: string), cmpltyp_cd (type: string), sum_reason_cd (type: string), svcrqct_cds (type: array), notes (type: array) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 - Statistics: Num rows: 1 Data size: 4944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 19776 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 - Estimated key counts: Map 1 => 1 + Estimated key counts: Map 1 => 2 keys: 0 _col0 (type: string) 1 _col0 (type: string) @@ -1254,17 +1243,17 @@ STAGE PLANS: input vertices: 0 Map 1 Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 3533 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 21753 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: array), _col10 (type: string), _col15 (type: array), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col11 (type: string), _col16 (type: array) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 - Statistics: Num rows: 1 Data size: 3533 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 21753 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 3533 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 21753 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1303,8 +1292,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.service_request_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct service_request_clean { string cnctevn_id, string svcrqst_id, string svcrqst_crt_dts, i32 subject_seq_no, string plan_component, string cust_segment, string cnctyp_cd, string cnctmd_cd, string cnctevs_cd, string svcrtyp_cd, string svrstyp_cd, string cmpltyp_cd, string catsrsn_cd, string apealvl_cd, string cnstnty_cd, string svcrqst_asrqst_ind, string svcrqst_rtnorig_in, string svcrqst_vwasof_dt, string sum_reason_cd, string sum_reason, string crsr_master_claim_index, list svcrqct_cds, string svcrqst_lupdt, timestamp crsr_lupdt, string cntevsds_lupdt, i32 ignore_me, list notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1327,8 +1314,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.service_request_clean numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct service_request_clean { string cnctevn_id, string svcrqst_id, string svcrqst_crt_dts, i32 subject_seq_no, string plan_component, string cust_segment, string cnctyp_cd, string cnctmd_cd, string cnctevs_cd, string svcrtyp_cd, string svrstyp_cd, string cmpltyp_cd, string catsrsn_cd, string apealvl_cd, string cnstnty_cd, string svcrqst_asrqst_ind, string svcrqst_rtnorig_in, string svcrqst_vwasof_dt, string sum_reason_cd, string sum_reason, string crsr_master_claim_index, list svcrqct_cds, string svcrqst_lupdt, timestamp crsr_lupdt, string cntevsds_lupdt, i32 ignore_me, list notes} serialization.format serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1353,7 +1338,8 @@ STAGE PLANS: name: default.ct_events1_test Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-0 diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out index 2ca78d7af8..5dfdede518 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition.q.out @@ -122,7 +122,8 @@ STAGE PLANS: name: default.partunion1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table partunion1 partition(part1) select temps.* from ( diff --git ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out index 78a17dfbc4..e7599a45f1 100644 --- ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_dynamic_partition_2.q.out @@ -144,7 +144,8 @@ STAGE PLANS: name: default.partunion1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Tez diff --git ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out index d399c5e4f5..f9535c50bb 100644 --- ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out +++ ql/src/test/results/clientpositive/llap/tez_union_multiinsert.q.out @@ -237,7 +237,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -250,7 +251,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( select key, value from ( @@ -1144,7 +1146,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1157,7 +1160,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( select key, value from src s0 @@ -2047,7 +2051,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2060,7 +2065,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( select key, value from src s0 @@ -2908,7 +2914,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2921,7 +2928,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION all @@ -3755,7 +3763,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3768,7 +3777,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION distinct diff --git ql/src/test/results/clientpositive/llap/union4.q.out ql/src/test/results/clientpositive/llap/union4.q.out index 796bc60a36..2716072320 100644 --- ql/src/test/results/clientpositive/llap/union4.q.out +++ ql/src/test/results/clientpositive/llap/union4.q.out @@ -134,7 +134,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/llap/union6.q.out ql/src/test/results/clientpositive/llap/union6.q.out index 4043e3cb87..b001433938 100644 --- ql/src/test/results/clientpositive/llap/union6.q.out +++ ql/src/test/results/clientpositive/llap/union6.q.out @@ -107,7 +107,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 diff --git ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out index 9401479abd..6b930c8a87 100644 --- ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out +++ ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out @@ -203,7 +203,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 @@ -664,7 +665,8 @@ STAGE PLANS: name: default.tmptable12 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable12 select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 @@ -2868,7 +2870,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2881,7 +2884,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT @@ -3696,7 +3700,8 @@ STAGE PLANS: name: default.dest118 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3709,7 +3714,8 @@ STAGE PLANS: name: default.dest218 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT @@ -4534,7 +4540,8 @@ STAGE PLANS: name: default.dest119 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4547,7 +4554,8 @@ STAGE PLANS: name: default.dest219 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION DISTINCT @@ -6813,7 +6821,8 @@ STAGE PLANS: name: default.dst_union22 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') @@ -9350,7 +9359,8 @@ STAGE PLANS: name: default.tmp_unionall Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -11025,7 +11035,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( @@ -11257,7 +11268,8 @@ STAGE PLANS: name: default.union_subq_union29 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union29 select * from ( @@ -11883,7 +11895,8 @@ STAGE PLANS: name: default.union_subq_union30 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union30 select * from ( @@ -12172,7 +12185,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -12185,7 +12199,8 @@ STAGE PLANS: name: default.t4 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from t1 @@ -12462,7 +12477,8 @@ STAGE PLANS: name: default.t5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -12475,7 +12491,8 @@ STAGE PLANS: name: default.t6 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( @@ -12776,7 +12793,8 @@ STAGE PLANS: name: default.t7 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -12789,7 +12807,8 @@ STAGE PLANS: name: default.t8 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( @@ -13756,7 +13775,8 @@ STAGE PLANS: name: default.test_src Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( @@ -13928,7 +13948,8 @@ STAGE PLANS: name: default.test_src Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( @@ -14631,7 +14652,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 @@ -14949,7 +14971,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 diff --git ql/src/test/results/clientpositive/llap/union_stats.q.out ql/src/test/results/clientpositive/llap/union_stats.q.out index 4178684db9..a492757bc1 100644 --- ql/src/test/results/clientpositive/llap/union_stats.q.out +++ ql/src/test/results/clientpositive/llap/union_stats.q.out @@ -202,7 +202,8 @@ STAGE PLANS: name: default.t Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-0 diff --git ql/src/test/results/clientpositive/llap/union_top_level.q.out ql/src/test/results/clientpositive/llap/union_top_level.q.out index cfbc069869..a010232171 100644 --- ql/src/test/results/clientpositive/llap/union_top_level.q.out +++ ql/src/test/results/clientpositive/llap/union_top_level.q.out @@ -562,7 +562,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -796,7 +797,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a @@ -1016,7 +1018,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a diff --git ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out index 6997af90d3..6c5f539282 100644 --- ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/llap/vector_auto_smb_mapjoin_14.q.out @@ -2031,7 +2031,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2044,7 +2045,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -2302,7 +2304,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2315,7 +2318,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/llap/vector_bucket.q.out ql/src/test/results/clientpositive/llap/vector_bucket.q.out index e6d57d6c15..ed15ff0d89 100644 --- ql/src/test/results/clientpositive/llap/vector_bucket.q.out +++ ql/src/test/results/clientpositive/llap/vector_bucket.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.non_orc_table Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: select a, b from non_orc_table order by a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_char_4.q.out ql/src/test/results/clientpositive/llap/vector_char_4.q.out index 0bf1a40424..52843389da 100644 --- ql/src/test/results/clientpositive/llap/vector_char_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_char_4.q.out @@ -195,5 +195,6 @@ STAGE PLANS: name: default.char_lazy_binary_columnar Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/vector_complex_all.q.out ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index 2f3f8867b1..d8dda5d362 100644 --- ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -838,7 +838,8 @@ STAGE PLANS: name: default.orc_create_complex Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 4' is a cross product PREHOOK: query: INSERT INTO TABLE orc_create_complex diff --git ql/src/test/results/clientpositive/llap/vector_groupby4.q.out ql/src/test/results/clientpositive/llap/vector_groupby4.q.out index bf2a366284..99ae3e67d9 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby4.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby4.q.out @@ -162,7 +162,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1) diff --git ql/src/test/results/clientpositive/llap/vector_groupby6.q.out ql/src/test/results/clientpositive/llap/vector_groupby6.q.out index 9fa46fb976..45c171b240 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby6.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby6.q.out @@ -162,7 +162,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1) diff --git ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out index 6b2ee48fdf..e2539b646b 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out @@ -884,7 +884,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -897,7 +898,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with cube diff --git ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out index 0e9d6a6c09..bc7b4dbfc8 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out @@ -573,7 +573,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -586,7 +587,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with rollup diff --git ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out index a25953f2a7..514f3fcdf5 100644 --- ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out +++ ql/src/test/results/clientpositive/llap/vector_multi_insert.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.orc_rn1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -182,7 +183,8 @@ STAGE PLANS: name: default.orc_rn2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -195,7 +197,8 @@ STAGE PLANS: name: default.orc_rn3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from orc1 a insert overwrite table orc_rn1 select a.* where a.rn < 100 diff --git ql/src/test/results/clientpositive/llap/vector_udf_character_length.q.out ql/src/test/results/clientpositive/llap/vector_udf_character_length.q.out index e596cf2aab..281a03de09 100644 --- ql/src/test/results/clientpositive/llap/vector_udf_character_length.q.out +++ ql/src/test/results/clientpositive/llap/vector_udf_character_length.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT character_length(src1.value) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_udf_octet_length.q.out ql/src/test/results/clientpositive/llap/vector_udf_octet_length.q.out index b156dc68e7..5b98553aa5 100644 --- ql/src/test/results/clientpositive/llap/vector_udf_octet_length.q.out +++ ql/src/test/results/clientpositive/llap/vector_udf_octet_length.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src1 INSERT OVERWRITE TABLE dest1 SELECT octet_length(src1.value) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out index a05e304214..b4a2d3177b 100644 --- ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/llap/vector_varchar_4.q.out @@ -195,5 +195,6 @@ STAGE PLANS: name: default.varchar_lazy_binary_columnar Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out index 7f0b9dae04..70e0d37979 100644 --- ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out +++ ql/src/test/results/clientpositive/llap/vector_varchar_simple.q.out @@ -410,7 +410,8 @@ STAGE PLANS: name: default.varchar_3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 16cae7927e..51aa716fad 100644 --- ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out index 1b2fc23235..bdd1138a6a 100644 --- ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out @@ -4021,7 +4021,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4034,7 +4035,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part_orc partition by p_mfgr diff --git ql/src/test/results/clientpositive/load_dyn_part1.q.out ql/src/test/results/clientpositive/load_dyn_part1.q.out index 43a8bc86c5..b43bd31128 100644 --- ql/src/test/results/clientpositive/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/load_dyn_part1.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -184,7 +185,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-10 Map Reduce diff --git ql/src/test/results/clientpositive/load_dyn_part10.q.out ql/src/test/results/clientpositive/load_dyn_part10.q.out index 066b3356dd..aea7798d0b 100644 --- ql/src/test/results/clientpositive/load_dyn_part10.q.out +++ ql/src/test/results/clientpositive/load_dyn_part10.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.nzhang_part10 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part10 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' diff --git ql/src/test/results/clientpositive/load_dyn_part13.q.out ql/src/test/results/clientpositive/load_dyn_part13.q.out index f8105c745b..8c9df3e9f2 100644 --- ql/src/test/results/clientpositive/load_dyn_part13.q.out +++ ql/src/test/results/clientpositive/load_dyn_part13.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.nzhang_part13 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part13 partition (ds="2010-03-03", hr) select * from ( diff --git ql/src/test/results/clientpositive/load_dyn_part14.q.out ql/src/test/results/clientpositive/load_dyn_part14.q.out index d65a1e3863..8e951b6e3e 100644 --- ql/src/test/results/clientpositive/load_dyn_part14.q.out +++ ql/src/test/results/clientpositive/load_dyn_part14.q.out @@ -145,7 +145,8 @@ STAGE PLANS: name: default.nzhang_part14 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/load_dyn_part2.q.out ql/src/test/results/clientpositive/load_dyn_part2.q.out index e8efa5f898..902212134e 100644 --- ql/src/test/results/clientpositive/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/load_dyn_part2.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.nzhang_part_bucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part_bucket partition (ds='2010-03-23', hr) select key, value, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/load_dyn_part3.q.out ql/src/test/results/clientpositive/load_dyn_part3.q.out index efe5222236..c63b4519f3 100644 --- ql/src/test/results/clientpositive/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/load_dyn_part3.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.nzhang_part3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part3 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/load_dyn_part4.q.out ql/src/test/results/clientpositive/load_dyn_part4.q.out index a561cc6f20..cf571921fc 100644 --- ql/src/test/results/clientpositive/load_dyn_part4.q.out +++ ql/src/test/results/clientpositive/load_dyn_part4.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.nzhang_part4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part4 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/load_dyn_part8.q.out ql/src/test/results/clientpositive/load_dyn_part8.q.out index cf2f60fbd9..9e20fdc921 100644 --- ql/src/test/results/clientpositive/load_dyn_part8.q.out +++ ql/src/test/results/clientpositive/load_dyn_part8.q.out @@ -363,7 +363,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -395,7 +396,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from srcpart diff --git ql/src/test/results/clientpositive/load_dyn_part9.q.out ql/src/test/results/clientpositive/load_dyn_part9.q.out index 25937dff8c..77e689ca2d 100644 --- ql/src/test/results/clientpositive/load_dyn_part9.q.out +++ ql/src/test/results/clientpositive/load_dyn_part9.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.nzhang_part9 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part9 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/mapreduce1.q.out ql/src/test/results/clientpositive/mapreduce1.q.out index 3d0a156557..602ae774af 100644 --- ql/src/test/results/clientpositive/mapreduce1.q.out +++ ql/src/test/results/clientpositive/mapreduce1.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce2.q.out ql/src/test/results/clientpositive/mapreduce2.q.out index 676c387c7a..a401ac0cc1 100644 --- ql/src/test/results/clientpositive/mapreduce2.q.out +++ ql/src/test/results/clientpositive/mapreduce2.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce3.q.out ql/src/test/results/clientpositive/mapreduce3.q.out index fc1a402b68..abc3f094c9 100644 --- ql/src/test/results/clientpositive/mapreduce3.q.out +++ ql/src/test/results/clientpositive/mapreduce3.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce4.q.out ql/src/test/results/clientpositive/mapreduce4.q.out index 17fa029ad4..91275a7108 100644 --- ql/src/test/results/clientpositive/mapreduce4.q.out +++ ql/src/test/results/clientpositive/mapreduce4.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce5.q.out ql/src/test/results/clientpositive/mapreduce5.q.out index 21103f88df..bb1e9e42b3 100644 --- ql/src/test/results/clientpositive/mapreduce5.q.out +++ ql/src/test/results/clientpositive/mapreduce5.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce6.q.out ql/src/test/results/clientpositive/mapreduce6.q.out index fe4e631077..1a31ebc1cb 100644 --- ql/src/test/results/clientpositive/mapreduce6.q.out +++ ql/src/test/results/clientpositive/mapreduce6.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce7.q.out ql/src/test/results/clientpositive/mapreduce7.q.out index cc97887fd8..b0c24fa8a8 100644 --- ql/src/test/results/clientpositive/mapreduce7.q.out +++ ql/src/test/results/clientpositive/mapreduce7.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/mapreduce8.q.out ql/src/test/results/clientpositive/mapreduce8.q.out index b1763c792f..fc866d558d 100644 --- ql/src/test/results/clientpositive/mapreduce8.q.out +++ ql/src/test/results/clientpositive/mapreduce8.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/masking_11.q.out ql/src/test/results/clientpositive/masking_11.q.out index f29c51fdd9..0941b8c2ab 100644 --- ql/src/test/results/clientpositive/masking_11.q.out +++ ql/src/test/results/clientpositive/masking_11.q.out @@ -13,8 +13,10 @@ POSTHOOK: Lineage: masking_test.value SIMPLE [(src)src.FieldSchema(name:value, t PREHOOK: query: analyze table `masking_test` compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@masking_test +PREHOOK: Output: default@masking_test #### A masked pattern was here #### POSTHOOK: query: analyze table `masking_test` compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test +POSTHOOK: Output: default@masking_test #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out index 8b67b55dff..de491989a5 100644 --- ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out +++ ql/src/test/results/clientpositive/materialized_view_rewrite_ssb.q.out @@ -187,42 +187,52 @@ POSTHOOK: Output: default@lineorder PREHOOK: query: analyze table customer compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@customer +PREHOOK: Output: default@customer #### A masked pattern was here #### POSTHOOK: query: analyze table customer compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@customer +POSTHOOK: Output: default@customer #### A masked pattern was here #### PREHOOK: query: analyze table dates compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@dates +PREHOOK: Output: default@dates #### A masked pattern was here #### POSTHOOK: query: analyze table dates compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@dates +POSTHOOK: Output: default@dates #### A masked pattern was here #### PREHOOK: query: analyze table ssb_part compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@ssb_part +PREHOOK: Output: default@ssb_part #### A masked pattern was here #### POSTHOOK: query: analyze table ssb_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@ssb_part +POSTHOOK: Output: default@ssb_part #### A masked pattern was here #### PREHOOK: query: analyze table supplier compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@supplier +PREHOOK: Output: default@supplier #### A masked pattern was here #### POSTHOOK: query: analyze table supplier compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@supplier +POSTHOOK: Output: default@supplier #### A masked pattern was here #### PREHOOK: query: analyze table lineorder compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@lineorder +PREHOOK: Output: default@lineorder #### A masked pattern was here #### POSTHOOK: query: analyze table lineorder compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@lineorder +POSTHOOK: Output: default@lineorder #### A masked pattern was here #### PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE AS diff --git ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out index 5e4c96dd7d..a11d66815a 100644 --- ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out +++ ql/src/test/results/clientpositive/materialized_view_rewrite_ssb_2.q.out @@ -187,42 +187,52 @@ POSTHOOK: Output: default@lineorder PREHOOK: query: analyze table customer compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@customer +PREHOOK: Output: default@customer #### A masked pattern was here #### POSTHOOK: query: analyze table customer compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@customer +POSTHOOK: Output: default@customer #### A masked pattern was here #### PREHOOK: query: analyze table dates compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@dates +PREHOOK: Output: default@dates #### A masked pattern was here #### POSTHOOK: query: analyze table dates compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@dates +POSTHOOK: Output: default@dates #### A masked pattern was here #### PREHOOK: query: analyze table ssb_part compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@ssb_part +PREHOOK: Output: default@ssb_part #### A masked pattern was here #### POSTHOOK: query: analyze table ssb_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@ssb_part +POSTHOOK: Output: default@ssb_part #### A masked pattern was here #### PREHOOK: query: analyze table supplier compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@supplier +PREHOOK: Output: default@supplier #### A masked pattern was here #### POSTHOOK: query: analyze table supplier compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@supplier +POSTHOOK: Output: default@supplier #### A masked pattern was here #### PREHOOK: query: analyze table lineorder compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@lineorder +PREHOOK: Output: default@lineorder #### A masked pattern was here #### POSTHOOK: query: analyze table lineorder compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@lineorder +POSTHOOK: Output: default@lineorder #### A masked pattern was here #### PREHOOK: query: CREATE MATERIALIZED VIEW `ssb_mv` ENABLE REWRITE AS diff --git ql/src/test/results/clientpositive/merge1.q.out ql/src/test/results/clientpositive/merge1.q.out index 7423d83ac8..bd8cd7bdfa 100644 --- ql/src/test/results/clientpositive/merge1.q.out +++ ql/src/test/results/clientpositive/merge1.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -548,7 +549,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -649,7 +651,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge2.q.out ql/src/test/results/clientpositive/merge2.q.out index bbc55d8b9e..0d781c29ce 100644 --- ql/src/test/results/clientpositive/merge2.q.out +++ ql/src/test/results/clientpositive/merge2.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -548,7 +549,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -649,7 +651,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge3.q.out ql/src/test/results/clientpositive/merge3.q.out index e05d651775..22a2820dd3 100644 --- ql/src/test/results/clientpositive/merge3.q.out +++ ql/src/test/results/clientpositive/merge3.q.out @@ -178,7 +178,8 @@ STAGE PLANS: name: default.merge_src2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -2558,7 +2559,8 @@ STAGE PLANS: name: default.merge_src_part2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -4993,7 +4995,8 @@ STAGE PLANS: name: default.merge_src_part2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/merge4.q.out ql/src/test/results/clientpositive/merge4.q.out index 182c6a887e..de4c593d53 100644 --- ql/src/test/results/clientpositive/merge4.q.out +++ ql/src/test/results/clientpositive/merge4.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.nzhang_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -1185,7 +1186,8 @@ STAGE PLANS: name: default.nzhang_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -2845,7 +2847,8 @@ STAGE PLANS: name: default.nzhang_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition.q.out ql/src/test/results/clientpositive/merge_dynamic_partition.q.out index a777fe0830..990e6dfb84 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table merge_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart_merge_dp where ds='2008-04-08' PREHOOK: type: QUERY @@ -692,7 +693,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -1322,7 +1324,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out index 5a2afb01ac..ccffc89d24 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out @@ -123,7 +123,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out index 055e07abd8..68d7bbe122 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out @@ -183,7 +183,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out index cbeaf42eaf..0da56cd9fc 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out @@ -184,7 +184,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out index 5a562f4456..45237a15b9 100644 --- ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out +++ ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out @@ -160,7 +160,8 @@ STAGE PLANS: name: default.merge_dynamic_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/metadata_only_queries.q.out ql/src/test/results/clientpositive/metadata_only_queries.q.out index 72a90fbf9d..0fe6684b82 100644 --- ql/src/test/results/clientpositive/metadata_only_queries.q.out +++ ql/src/test/results/clientpositive/metadata_only_queries.q.out @@ -395,40 +395,54 @@ STAGE PLANS: PREHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl +PREHOOK: Output: default@stats_tbl #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl +POSTHOOK: Output: default@stats_tbl #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2011 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2011 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2012 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2012 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### PREHOOK: query: explain select count(*), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si) from stats_tbl diff --git ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out index 6376aa79f8..79d9d27203 100644 --- ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out +++ ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out @@ -126,21 +126,29 @@ PREHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2014 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2014 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### PREHOOK: query: explain select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010 diff --git ql/src/test/results/clientpositive/mm_all.q.out ql/src/test/results/clientpositive/mm_all.q.out index c943f0192c..0e04978abd 100644 --- ql/src/test/results/clientpositive/mm_all.q.out +++ ql/src/test/results/clientpositive/mm_all.q.out @@ -105,7 +105,8 @@ STAGE PLANS: micromanaged table: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/multi_insert_gby.q.out ql/src/test/results/clientpositive/multi_insert_gby.q.out index 0ed05ddb97..85fd8b7f5e 100644 --- ql/src/test/results/clientpositive/multi_insert_gby.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -118,7 +119,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 @@ -284,7 +286,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -297,7 +300,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/multi_insert_gby2.q.out ql/src/test/results/clientpositive/multi_insert_gby2.q.out index d1da4e75cd..ceb694052f 100644 --- ql/src/test/results/clientpositive/multi_insert_gby2.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby2.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -108,7 +109,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as value from src order by key) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/multi_insert_gby3.q.out ql/src/test/results/clientpositive/multi_insert_gby3.q.out index 610b15fd1d..d004e644e9 100644 --- ql/src/test/results/clientpositive/multi_insert_gby3.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby3.q.out @@ -111,7 +111,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -124,7 +125,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -215,7 +217,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -228,7 +231,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a INSERT OVERWRITE TABLE e1 @@ -1635,7 +1639,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -1677,7 +1682,8 @@ STAGE PLANS: name: default.e3 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -1787,7 +1793,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1800,7 +1807,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -1842,5 +1850,6 @@ STAGE PLANS: name: default.e3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/multi_insert_gby4.q.out ql/src/test/results/clientpositive/multi_insert_gby4.q.out index 2f59665626..2b464d36a7 100644 --- ql/src/test/results/clientpositive/multi_insert_gby4.q.out +++ ql/src/test/results/clientpositive/multi_insert_gby4.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -159,7 +160,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -172,7 +174,8 @@ STAGE PLANS: name: default.e3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (SELECT key, value FROM src) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/multi_insert_mixed.q.out ql/src/test/results/clientpositive/multi_insert_mixed.q.out index 9acae2ec54..05c030e0e0 100644 --- ql/src/test/results/clientpositive/multi_insert_mixed.q.out +++ ql/src/test/results/clientpositive/multi_insert_mixed.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -208,7 +209,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -221,7 +223,8 @@ STAGE PLANS: name: default.src_multi3 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select key, count(1) group by key order by key diff --git ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out index 3a8ec94559..281a05aeb8 100644 --- ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out +++ ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -97,7 +98,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -241,7 +243,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -254,7 +257,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -448,7 +452,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -461,7 +466,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -605,7 +611,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -618,7 +625,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -825,7 +833,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -838,7 +847,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -988,7 +998,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1001,7 +1012,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -1201,7 +1213,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1214,7 +1227,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1364,7 +1378,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1377,7 +1392,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -1593,7 +1609,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1606,7 +1623,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1798,7 +1816,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1811,7 +1830,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -2053,7 +2073,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2066,7 +2087,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -2258,7 +2280,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2271,7 +2294,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -3374,7 +3398,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3387,7 +3412,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3598,7 +3624,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3611,7 +3638,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3841,7 +3869,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3854,7 +3883,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-12 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce @@ -4153,7 +4183,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4166,7 +4197,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-12 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce diff --git ql/src/test/results/clientpositive/multi_insert_union_src.q.out ql/src/test/results/clientpositive/multi_insert_union_src.q.out index 1ff1db5c12..f7f5d7510d 100644 --- ql/src/test/results/clientpositive/multi_insert_union_src.q.out +++ ql/src/test/results/clientpositive/multi_insert_union_src.q.out @@ -143,7 +143,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -179,7 +180,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src1 where key < 10 union all select * from src2 where key > 100) s insert overwrite table src_multi1 select key, value where key < 150 order by key diff --git ql/src/test/results/clientpositive/multi_insert_with_join2.q.out ql/src/test/results/clientpositive/multi_insert_with_join2.q.out index 0231318ab7..a554215123 100644 --- ql/src/test/results/clientpositive/multi_insert_with_join2.q.out +++ ql/src/test/results/clientpositive/multi_insert_with_join2.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain @@ -212,7 +213,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a LEFT JOIN T_B b ON a.id = b.id @@ -312,7 +314,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -325,7 +328,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a LEFT JOIN T_B b ON a.id = b.id @@ -425,7 +429,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -438,7 +443,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -544,7 +550,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -557,7 +564,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -663,7 +671,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -676,7 +685,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -794,7 +804,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -807,7 +818,8 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM T_A a JOIN T_B b ON a.id = b.id @@ -925,7 +937,8 @@ STAGE PLANS: name: default.join_result_1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -938,5 +951,6 @@ STAGE PLANS: name: default.join_result_3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/multigroupby_singlemr.q.out ql/src/test/results/clientpositive/multigroupby_singlemr.q.out index 7af8c4356d..cd030e55ab 100644 --- ql/src/test/results/clientpositive/multigroupby_singlemr.q.out +++ ql/src/test/results/clientpositive/multigroupby_singlemr.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -169,7 +170,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -259,7 +261,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -302,7 +305,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -392,7 +396,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -435,7 +440,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -522,7 +528,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -535,7 +542,8 @@ STAGE PLANS: name: default.dest4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -646,7 +654,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -689,7 +698,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-7 Map Reduce @@ -732,5 +742,6 @@ STAGE PLANS: name: default.dest1 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/nonmr_fetch.q.out ql/src/test/results/clientpositive/nonmr_fetch.q.out index d2144b111b..dffa80d05a 100644 --- ql/src/test/results/clientpositive/nonmr_fetch.q.out +++ ql/src/test/results/clientpositive/nonmr_fetch.q.out @@ -1090,7 +1090,8 @@ STAGE PLANS: name: default.srcx Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain analyze table src compute statistics PREHOOK: type: QUERY @@ -1109,7 +1110,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain select * from src join src src2 on src.key=src2.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out index d01461b51f..d6a8da5b15 100644 --- ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out +++ ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 PREHOOK: type: QUERY @@ -155,7 +156,8 @@ STAGE PLANS: name: default.insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE `insert` SELECT * FROM src LIMIT 100 PREHOOK: type: QUERY @@ -249,7 +251,8 @@ STAGE PLANS: name: default.insert Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE `insert` SELECT * FROM src LIMIT 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/notable_alias1.q.out ql/src/test/results/clientpositive/notable_alias1.q.out index 677545d8d2..ff1778b66b 100644 --- ql/src/test/results/clientpositive/notable_alias1.q.out +++ ql/src/test/results/clientpositive/notable_alias1.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT '1234', key, count(1) WHERE src.key < 100 group by key diff --git ql/src/test/results/clientpositive/notable_alias2.q.out ql/src/test/results/clientpositive/notable_alias2.q.out index 66d0b2a92a..2f8b0e9b27 100644 --- ql/src/test/results/clientpositive/notable_alias2.q.out +++ ql/src/test/results/clientpositive/notable_alias2.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT '1234', src.key, count(1) WHERE key < 100 group by src.key diff --git ql/src/test/results/clientpositive/nullformatCTAS.q.out ql/src/test/results/clientpositive/nullformatCTAS.q.out index cda09658ce..723a4c9ac1 100644 --- ql/src/test/results/clientpositive/nullformatCTAS.q.out +++ ql/src/test/results/clientpositive/nullformatCTAS.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.null_tab3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/optimize_filter_literal.q.out ql/src/test/results/clientpositive/optimize_filter_literal.q.out index 32e2bf6415..ad7e0a418d 100644 --- ql/src/test/results/clientpositive/optimize_filter_literal.q.out +++ ql/src/test/results/clientpositive/optimize_filter_literal.q.out @@ -90,11 +90,15 @@ PREHOOK: query: analyze table tab_part partition (ds='2008-04-08') compute stati PREHOOK: type: QUERY PREHOOK: Input: default@tab_part PREHOOK: Input: default@tab_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part +PREHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab_part partition (ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part +POSTHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS ORCFILE PREHOOK: type: CREATETABLE @@ -122,11 +126,15 @@ PREHOOK: query: analyze table tab partition (ds='2008-04-08') compute statistics PREHOOK: type: QUERY PREHOOK: Input: default@tab PREHOOK: Input: default@tab@ds=2008-04-08 +PREHOOK: Output: default@tab +PREHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab partition (ds='2008-04-08') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab POSTHOOK: Input: default@tab@ds=2008-04-08 +POSTHOOK: Output: default@tab +POSTHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### Warning: Shuffle Join JOIN[12][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: select * from diff --git ql/src/test/results/clientpositive/orc_createas1.q.out ql/src/test/results/clientpositive/orc_createas1.q.out index 506f39d59f..ecfad35286 100644 --- ql/src/test/results/clientpositive/orc_createas1.q.out +++ ql/src/test/results/clientpositive/orc_createas1.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.orc_createas1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -274,7 +275,8 @@ STAGE PLANS: name: default.orc_createas1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge1.q.out ql/src/test/results/clientpositive/orc_merge1.q.out index a83e85bec4..936e93fc06 100644 --- ql/src/test/results/clientpositive/orc_merge1.q.out +++ ql/src/test/results/clientpositive/orc_merge1.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -173,7 +174,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge10.q.out ql/src/test/results/clientpositive/orc_merge10.q.out index 607aaeb6ae..94956e8565 100644 --- ql/src/test/results/clientpositive/orc_merge10.q.out +++ ql/src/test/results/clientpositive/orc_merge10.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -173,7 +174,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -418,7 +421,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: ALTER TABLE orcfile_merge1 PARTITION (ds='1', part='0') CONCATENATE PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/orc_merge2.q.out ql/src/test/results/clientpositive/orc_merge2.q.out index d4c474f9fe..5e8a1f0ccb 100644 --- ql/src/test/results/clientpositive/orc_merge2.q.out +++ ql/src/test/results/clientpositive/orc_merge2.q.out @@ -78,7 +78,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge3.q.out ql/src/test/results/clientpositive/orc_merge3.q.out index 7bf12c6c28..e527f11795 100644 --- ql/src/test/results/clientpositive/orc_merge3.q.out +++ ql/src/test/results/clientpositive/orc_merge3.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge4.q.out ql/src/test/results/clientpositive/orc_merge4.q.out index 828f204157..c063fc345e 100644 --- ql/src/test/results/clientpositive/orc_merge4.q.out +++ ql/src/test/results/clientpositive/orc_merge4.q.out @@ -122,7 +122,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/orc_merge5.q.out ql/src/test/results/clientpositive/orc_merge5.q.out index fe24e16a58..069b857998 100644 --- ql/src/test/results/clientpositive/orc_merge5.q.out +++ ql/src/test/results/clientpositive/orc_merge5.q.out @@ -66,7 +66,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -160,7 +161,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -273,7 +275,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5b concatenate PREHOOK: type: ALTER_TABLE_MERGE diff --git ql/src/test/results/clientpositive/orc_merge6.q.out ql/src/test/results/clientpositive/orc_merge6.q.out index d998f4c388..8d11589e8d 100644 --- ql/src/test/results/clientpositive/orc_merge6.q.out +++ ql/src/test/results/clientpositive/orc_merge6.q.out @@ -69,7 +69,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (year="2000",hour=24) select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -208,7 +209,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -408,7 +410,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(year="2000",hour=24) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out index a83e85bec4..936e93fc06 100644 --- ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out @@ -89,7 +89,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -173,7 +174,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/orc_merge_incompat1.q.out index aac4d1589e..01c9df85a1 100644 --- ql/src/test/results/clientpositive/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/orc_merge_incompat1.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/orc_merge_incompat2.q.out index 4b36e972a9..86d5d3c1a2 100644 --- ql/src/test/results/clientpositive/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/orc_merge_incompat2.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 order by userid PREHOOK: type: QUERY @@ -277,7 +278,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/outer_reference_windowed.q.out ql/src/test/results/clientpositive/outer_reference_windowed.q.out index 06d814573e..aef80c5aed 100644 --- ql/src/test/results/clientpositive/outer_reference_windowed.q.out +++ ql/src/test/results/clientpositive/outer_reference_windowed.q.out @@ -93,27 +93,67 @@ POSTHOOK: Lineage: e011_03.c2 SIMPLE [(e011_01)e011_01.FieldSchema(name:c2, type PREHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_01 +PREHOOK: Output: default@e011_01 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_01 +POSTHOOK: Output: default@e011_01 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_02 +PREHOOK: Output: default@e011_02 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_02 +POSTHOOK: Output: default@e011_02 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_03 +PREHOOK: Output: default@e011_03 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_03 +POSTHOOK: Output: default@e011_03 +#### A masked pattern was here #### +PREHOOK: query: describe formatted e011_01 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@e011_01 +POSTHOOK: query: describe formatted e011_01 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@e011_01 +# col_name data_type comment +c1 decimal(15,2) +c2 decimal(15,2) + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\"}} + numFiles 1 + numRows 4 + rawDataSize 12 + totalSize 16 #### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 PREHOOK: query: explain select sum(sum(c1)) over() from e011_01 PREHOOK: type: QUERY POSTHOOK: query: explain select sum(sum(c1)) over() from e011_01 @@ -129,11 +169,11 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)) outputColumnNames: c1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) mode: hash @@ -239,22 +279,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: c1, c2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) keys: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Group By Operator @@ -262,7 +302,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -278,13 +318,13 @@ STAGE PLANS: key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col1 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -304,14 +344,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -365,19 +405,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -402,13 +442,13 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) keys: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -424,7 +464,7 @@ STAGE PLANS: key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Group By Operator @@ -432,7 +472,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -448,13 +488,13 @@ STAGE PLANS: key expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col1 (type: decimal(15,2)) - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -474,14 +514,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -555,19 +595,19 @@ STAGE PLANS: value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: @@ -712,19 +752,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -750,13 +790,13 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: corr(_col0, _col2) keys: _col1 (type: decimal(15,2)), _col3 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -772,7 +812,7 @@ STAGE PLANS: key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct) Reduce Operator Tree: Group By Operator @@ -780,11 +820,11 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)), _col2 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -804,14 +844,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumDouble window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/parallel.q.out ql/src/test/results/clientpositive/parallel.q.out index 459105e09a..31b1a0dd3d 100644 --- ql/src/test/results/clientpositive/parallel.q.out +++ ql/src/test/results/clientpositive/parallel.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -129,7 +130,8 @@ STAGE PLANS: name: default.src_b Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/parallel_colstats.q.out ql/src/test/results/clientpositive/parallel_colstats.q.out index 83fc14a72f..12ad5ee874 100644 --- ql/src/test/results/clientpositive/parallel_colstats.q.out +++ ql/src/test/results/clientpositive/parallel_colstats.q.out @@ -28,12 +28,10 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 Stage-0 depends on stages: Stage-3 - Stage-4 depends on stages: Stage-0 - Stage-8 depends on stages: Stage-4, Stage-5, Stage-6, Stage-7 - Stage-9 depends on stages: Stage-4, Stage-5, Stage-6, Stage-7 + Stage-4 depends on stages: Stage-0, Stage-5, Stage-7 Stage-5 depends on stages: Stage-3 + Stage-6 depends on stages: Stage-1, Stage-5, Stage-7 Stage-1 depends on stages: Stage-3 - Stage-6 depends on stages: Stage-1 Stage-7 depends on stages: Stage-3 STAGE PLANS: @@ -150,22 +148,13 @@ STAGE PLANS: name: default.src_a Stage: Stage-4 - Stats-Aggr Operator - - Stage: Stage-8 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: string, string Table: default.src_a - Stage: Stage-9 - Column Stats Work - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.src_b - Stage: Stage-5 Map Reduce Map Operator Tree: @@ -188,6 +177,14 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Stage: Stage-6 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.src_b + Stage: Stage-1 Move Operator tables: @@ -198,9 +195,6 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.src_b - Stage: Stage-6 - Stats-Aggr Operator - Stage: Stage-7 Map Reduce Map Operator Tree: diff --git ql/src/test/results/clientpositive/parallel_join1.q.out ql/src/test/results/clientpositive/parallel_join1.q.out index 8843661176..0450e1586d 100644 --- ql/src/test/results/clientpositive/parallel_join1.q.out +++ ql/src/test/results/clientpositive/parallel_join1.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/parallel_orderby.q.out ql/src/test/results/clientpositive/parallel_orderby.q.out index 699dc16aa6..3a80dfc8cc 100644 --- ql/src/test/results/clientpositive/parallel_orderby.q.out +++ ql/src/test/results/clientpositive/parallel_orderby.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.total_ordered Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table total_ordered as select * from src5 order by key, value PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/parquet_mixed_partition_formats.q.out ql/src/test/results/clientpositive/parquet_mixed_partition_formats.q.out index e931aaa335..075dd4c682 100644 --- ql/src/test/results/clientpositive/parquet_mixed_partition_formats.q.out +++ ql/src/test/results/clientpositive/parquet_mixed_partition_formats.q.out @@ -122,8 +122,6 @@ Table: parquet_mixed_partition_formats #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 2521 #### A masked pattern was here #### @@ -243,8 +241,6 @@ Table: parquet_mixed_partition_formats #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 2521 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/parquet_serde.q.out ql/src/test/results/clientpositive/parquet_serde.q.out index 90dc6a5ef2..b6efd42b39 100644 --- ql/src/test/results/clientpositive/parquet_serde.q.out +++ ql/src/test/results/clientpositive/parquet_serde.q.out @@ -66,8 +66,6 @@ Table: parquet_mixed_fileformat #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 36 #### A masked pattern was here #### @@ -167,8 +165,6 @@ Table: parquet_mixed_fileformat #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 36 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/partial_column_stats.q.out ql/src/test/results/clientpositive/partial_column_stats.q.out index c1a78c38c6..44db81a443 100644 --- ql/src/test/results/clientpositive/partial_column_stats.q.out +++ ql/src/test/results/clientpositive/partial_column_stats.q.out @@ -49,7 +49,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: key, value Column Types: int, string @@ -58,10 +59,12 @@ STAGE PLANS: PREHOOK: query: analyze table t1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 #### A masked pattern was here #### POSTHOOK: query: analyze table t1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 #### A masked pattern was here #### PREHOOK: query: desc formatted t1 value PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/partition_coltype_literals.q.out ql/src/test/results/clientpositive/partition_coltype_literals.q.out index 9a0b4f423f..adabbb2e10 100644 --- ql/src/test/results/clientpositive/partition_coltype_literals.q.out +++ ql/src/test/results/clientpositive/partition_coltype_literals.q.out @@ -281,11 +281,15 @@ PREHOOK: query: analyze table partcoltypenum partition (tint=110Y, sint=22000S, PREHOOK: type: QUERY PREHOOK: Input: default@partcoltypenum PREHOOK: Input: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 +PREHOOK: Output: default@partcoltypenum +PREHOOK: Output: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 #### A masked pattern was here #### POSTHOOK: query: analyze table partcoltypenum partition (tint=110Y, sint=22000S, bint=330000000000L) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcoltypenum POSTHOOK: Input: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 +POSTHOOK: Output: default@partcoltypenum +POSTHOOK: Output: default@partcoltypenum@tint=110/sint=22000/bint=330000000000 #### A masked pattern was here #### PREHOOK: query: describe formatted partcoltypenum partition (tint=110Y, sint=22000S, bint=330000000000L) key PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/pcr.q.out ql/src/test/results/clientpositive/pcr.q.out index 691048c1fd..3f69f8bf19 100644 --- ql/src/test/results/clientpositive/pcr.q.out +++ ql/src/test/results/clientpositive/pcr.q.out @@ -3649,7 +3649,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -3870,7 +3871,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 @@ -4279,7 +4281,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 @@ -4500,7 +4503,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-10 diff --git ql/src/test/results/clientpositive/pcs.q.out ql/src/test/results/clientpositive/pcs.q.out index cc6ba5afef..bbda3829e9 100644 --- ql/src/test/results/clientpositive/pcs.q.out +++ ql/src/test/results/clientpositive/pcs.q.out @@ -70,6 +70,10 @@ PREHOOK: Input: default@pcs_t1 PREHOOK: Input: default@pcs_t1@ds=2000-04-08 PREHOOK: Input: default@pcs_t1@ds=2000-04-09 PREHOOK: Input: default@pcs_t1@ds=2000-04-10 +PREHOOK: Output: default@pcs_t1 +PREHOOK: Output: default@pcs_t1@ds=2000-04-08 +PREHOOK: Output: default@pcs_t1@ds=2000-04-09 +PREHOOK: Output: default@pcs_t1@ds=2000-04-10 #### A masked pattern was here #### POSTHOOK: query: analyze table pcs_t1 partition(ds) compute statistics for columns POSTHOOK: type: QUERY @@ -77,6 +81,10 @@ POSTHOOK: Input: default@pcs_t1 POSTHOOK: Input: default@pcs_t1@ds=2000-04-08 POSTHOOK: Input: default@pcs_t1@ds=2000-04-09 POSTHOOK: Input: default@pcs_t1@ds=2000-04-10 +POSTHOOK: Output: default@pcs_t1 +POSTHOOK: Output: default@pcs_t1@ds=2000-04-08 +POSTHOOK: Output: default@pcs_t1@ds=2000-04-09 +POSTHOOK: Output: default@pcs_t1@ds=2000-04-10 #### A masked pattern was here #### PREHOOK: query: explain extended select key, value, ds from pcs_t1 where (ds='2000-04-08' and key=1) or (ds='2000-04-09' and key=2) order by key, value, ds PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/ppd_constant_expr.q.out ql/src/test/results/clientpositive/ppd_constant_expr.q.out index cbe76549f7..ca35c4c393 100644 --- ql/src/test/results/clientpositive/ppd_constant_expr.q.out +++ ql/src/test/results/clientpositive/ppd_constant_expr.q.out @@ -64,7 +64,8 @@ STAGE PLANS: name: default.ppd_constant_expr Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -200,7 +201,8 @@ STAGE PLANS: name: default.ppd_constant_expr Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/ppd_multi_insert.q.out ql/src/test/results/clientpositive/ppd_multi_insert.q.out index ab50ea1276..60b17ecfbd 100644 --- ql/src/test/results/clientpositive/ppd_multi_insert.q.out +++ ql/src/test/results/clientpositive/ppd_multi_insert.q.out @@ -161,7 +161,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -174,7 +175,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -190,7 +192,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1426,7 +1429,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1439,7 +1443,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1455,7 +1460,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator diff --git ql/src/test/results/clientpositive/quote1.q.out ql/src/test/results/clientpositive/quote1.q.out index f873dc34c6..4a4fbd0ab7 100644 --- ql/src/test/results/clientpositive/quote1.q.out +++ ql/src/test/results/clientpositive/quote1.q.out @@ -69,7 +69,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/quotedid_stats.q.out ql/src/test/results/clientpositive/quotedid_stats.q.out index 418f7f2e5a..b82543e4c3 100644 --- ql/src/test/results/clientpositive/quotedid_stats.q.out +++ ql/src/test/results/clientpositive/quotedid_stats.q.out @@ -43,10 +43,12 @@ Storage Desc Params: PREHOOK: query: analyze table t4 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@t4 +PREHOOK: Output: default@t4 #### A masked pattern was here #### POSTHOOK: query: analyze table t4 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@t4 +POSTHOOK: Output: default@t4 #### A masked pattern was here #### PREHOOK: query: describe formatted t4 PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/rand_partitionpruner2.q.out ql/src/test/results/clientpositive/rand_partitionpruner2.q.out index 9b29136ba1..1482fde260 100644 --- ql/src/test/results/clientpositive/rand_partitionpruner2.q.out +++ ql/src/test/results/clientpositive/rand_partitionpruner2.q.out @@ -213,7 +213,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/rcfile_null_value.q.out ql/src/test/results/clientpositive/rcfile_null_value.q.out index c6a4a18370..e2a8ae0c4e 100644 --- ql/src/test/results/clientpositive/rcfile_null_value.q.out +++ ql/src/test/results/clientpositive/rcfile_null_value.q.out @@ -161,7 +161,8 @@ STAGE PLANS: name: default.dest1_rc Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/remove_exprs_stats.q.out ql/src/test/results/clientpositive/remove_exprs_stats.q.out index c256a5cab5..8f72a469f5 100644 --- ql/src/test/results/clientpositive/remove_exprs_stats.q.out +++ ql/src/test/results/clientpositive/remove_exprs_stats.q.out @@ -55,10 +55,12 @@ POSTHOOK: Lineage: loc_orc.zip SIMPLE [(loc_staging)loc_staging.FieldSchema(name PREHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc +PREHOOK: Output: default@loc_orc #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc +POSTHOOK: Output: default@loc_orc #### A masked pattern was here #### PREHOOK: query: explain select * from loc_orc where locid < 30 PREHOOK: type: QUERY @@ -594,10 +596,12 @@ POSTHOOK: Lineage: t.s SIMPLE [(values__tmp__table__1)values__tmp__table__1.Fiel PREHOOK: query: analyze table t compute statistics for columns s PREHOOK: type: QUERY PREHOOK: Input: default@t +PREHOOK: Output: default@t #### A masked pattern was here #### POSTHOOK: query: analyze table t compute statistics for columns s POSTHOOK: type: QUERY POSTHOOK: Input: default@t +POSTHOOK: Output: default@t #### A masked pattern was here #### PREHOOK: query: explain select * from t where s is null PREHOOK: type: QUERY @@ -723,10 +727,12 @@ POSTHOOK: Lineage: t.s SIMPLE [(values__tmp__table__2)values__tmp__table__2.Fiel PREHOOK: query: analyze table t compute statistics for columns s PREHOOK: type: QUERY PREHOOK: Input: default@t +PREHOOK: Output: default@t #### A masked pattern was here #### POSTHOOK: query: analyze table t compute statistics for columns s POSTHOOK: type: QUERY POSTHOOK: Input: default@t +POSTHOOK: Output: default@t #### A masked pattern was here #### PREHOOK: query: explain select * from t where s is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/rename_external_partition_location.q.out ql/src/test/results/clientpositive/rename_external_partition_location.q.out index def0aae6e3..8e3c17728e 100644 --- ql/src/test/results/clientpositive/rename_external_partition_location.q.out +++ ql/src/test/results/clientpositive/rename_external_partition_location.q.out @@ -69,11 +69,15 @@ PREHOOK: query: ANALYZE TABLE ex_table PARTITION (part='part1') COMPUTE STATISTI PREHOOK: type: QUERY PREHOOK: Input: default@ex_table PREHOOK: Input: default@ex_table@part=part1 +PREHOOK: Output: default@ex_table +PREHOOK: Output: default@ex_table@part=part1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE ex_table PARTITION (part='part1') COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@ex_table POSTHOOK: Input: default@ex_table@part=part1 +POSTHOOK: Output: default@ex_table +POSTHOOK: Output: default@ex_table@part=part1 #### A masked pattern was here #### PREHOOK: query: DESCRIBE FORMATTED ex_table PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out index e29bba4482..b0f9069407 100644 --- ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out +++ ql/src/test/results/clientpositive/rename_table_update_column_stats.q.out @@ -47,10 +47,12 @@ PREHOOK: query: analyze table testtable1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable1 POSTHOOK: query: analyze table testtable1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable1 PREHOOK: query: describe formatted statsdb1.testtable1 col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable1 @@ -247,10 +249,12 @@ PREHOOK: query: analyze table testtable1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable1 POSTHOOK: query: analyze table testtable1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable1 PREHOOK: query: describe formatted statsdb1.testtable1 col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable1 diff --git ql/src/test/results/clientpositive/sample1.q.out ql/src/test/results/clientpositive/sample1.q.out index dec9b233b3..72c32b4506 100644 --- ql/src/test/results/clientpositive/sample1.q.out +++ ql/src/test/results/clientpositive/sample1.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample2.q.out ql/src/test/results/clientpositive/sample2.q.out index f54c57363a..e09fd50225 100644 --- ql/src/test/results/clientpositive/sample2.q.out +++ ql/src/test/results/clientpositive/sample2.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample4.q.out ql/src/test/results/clientpositive/sample4.q.out index 675fda9ef4..46a1c6f425 100644 --- ql/src/test/results/clientpositive/sample4.q.out +++ ql/src/test/results/clientpositive/sample4.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample5.q.out ql/src/test/results/clientpositive/sample5.q.out index 583784e9b7..15e89d0d98 100644 --- ql/src/test/results/clientpositive/sample5.q.out +++ ql/src/test/results/clientpositive/sample5.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample6.q.out ql/src/test/results/clientpositive/sample6.q.out index 36e6906785..c568bd2742 100644 --- ql/src/test/results/clientpositive/sample6.q.out +++ ql/src/test/results/clientpositive/sample6.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/sample7.q.out ql/src/test/results/clientpositive/sample7.q.out index f0d9088174..6ad8ced7d1 100644 --- ql/src/test/results/clientpositive/sample7.q.out +++ ql/src/test/results/clientpositive/sample7.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/skewjoin.q.out ql/src/test/results/clientpositive/skewjoin.q.out index 9df48edeca..d349f98448 100644 --- ql/src/test/results/clientpositive/skewjoin.q.out +++ ql/src/test/results/clientpositive/skewjoin.q.out @@ -198,7 +198,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/skewjoin_noskew.q.out ql/src/test/results/clientpositive/skewjoin_noskew.q.out index 243a840406..d39acaf10a 100644 --- ql/src/test/results/clientpositive/skewjoin_noskew.q.out +++ ql/src/test/results/clientpositive/skewjoin_noskew.q.out @@ -146,7 +146,8 @@ STAGE PLANS: name: default.noskew Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table noskew as select a.* from src a join src b on a.key=b.key order by a.key limit 30 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out index 09aaaf684a..a5c0df9aa7 100644 --- ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out +++ ql/src/test/results/clientpositive/skewjoin_onesideskew.q.out @@ -186,7 +186,8 @@ STAGE PLANS: name: default.result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TABLE result AS SELECT a.* FROM skewtable a JOIN nonskewtable b ON a.key=b.key PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/smb_mapjoin9.q.out ql/src/test/results/clientpositive/smb_mapjoin9.q.out index 4ba3c28ff3..d1d0314b2c 100644 --- ql/src/test/results/clientpositive/smb_mapjoin9.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin9.q.out @@ -280,7 +280,8 @@ STAGE PLANS: name: default.smb_mapjoin9_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/smb_mapjoin_1.q.out ql/src/test/results/clientpositive/smb_mapjoin_1.q.out index b3f36ad580..9e20b84244 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_1.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_1.q.out @@ -46,6 +46,47 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 +PREHOOK: query: desc formatted smb_bucket_1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@smb_bucket_1 +POSTHOOK: query: desc formatted smb_bucket_1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@smb_bucket_1 +# col_name data_type comment +key int +value string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + SORTBUCKETCOLSPREFIX TRUE + numFiles 1 + totalSize 208 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [key] +Sort Columns: [Order(col:key, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select count(*) from smb_bucket_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@smb_bucket_1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from smb_bucket_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@smb_bucket_1 +#### A masked pattern was here #### +5 PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key PREHOOK: type: QUERY @@ -283,10 +324,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -336,7 +377,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Left Outer Join 0 to 1 @@ -391,7 +432,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Right Outer Join 0 to 1 @@ -445,7 +486,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Outer Join 0 to 1 diff --git ql/src/test/results/clientpositive/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/smb_mapjoin_11.q.out index b53e6704cc..b76e7a8221 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_11.q.out @@ -202,7 +202,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' diff --git ql/src/test/results/clientpositive/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/smb_mapjoin_12.q.out index 9af4683451..6def3de813 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_12.q.out @@ -204,7 +204,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' @@ -409,7 +410,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') diff --git ql/src/test/results/clientpositive/smb_mapjoin_20.q.out ql/src/test/results/clientpositive/smb_mapjoin_20.q.out index 48e4d05a66..ac62af2966 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_20.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_20.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value, a.value FROM test_table1 a WHERE a.ds = '1' @@ -202,7 +203,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.value, a.key, a.value FROM test_table1 a WHERE a.ds = '1' @@ -1364,5 +1366,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/smb_mapjoin_21.q.out ql/src/test/results/clientpositive/smb_mapjoin_21.q.out index c0fdfd38d2..8b568eb258 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_21.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_21.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -153,7 +154,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -229,7 +231,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -306,7 +309,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -383,7 +387,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -459,5 +464,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/smb_mapjoin_22.q.out ql/src/test/results/clientpositive/smb_mapjoin_22.q.out index 36e879236f..256a3c26e0 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_22.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_22.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 @@ -212,7 +213,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 diff --git ql/src/test/results/clientpositive/smb_mapjoin_7.q.out ql/src/test/results/clientpositive/smb_mapjoin_7.q.out index 82f5804eea..a2a8660c3b 100644 --- ql/src/test/results/clientpositive/smb_mapjoin_7.q.out +++ ql/src/test/results/clientpositive/smb_mapjoin_7.q.out @@ -669,7 +669,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out index 02d1605f40..118a48eb57 100644 --- ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out +++ ql/src/test/results/clientpositive/spark/annotate_stats_join.q.out @@ -97,26 +97,32 @@ POSTHOOK: Output: default@loc PREHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid PREHOOK: type: QUERY PREHOOK: Input: default@emp +PREHOOK: Output: default@emp #### A masked pattern was here #### POSTHOOK: query: analyze table emp compute statistics for columns lastname,deptid,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@emp +POSTHOOK: Output: default@emp #### A masked pattern was here #### PREHOOK: query: analyze table dept compute statistics for columns deptname,deptid PREHOOK: type: QUERY PREHOOK: Input: default@dept +PREHOOK: Output: default@dept #### A masked pattern was here #### POSTHOOK: query: analyze table dept compute statistics for columns deptname,deptid POSTHOOK: type: QUERY POSTHOOK: Input: default@dept +POSTHOOK: Output: default@dept #### A masked pattern was here #### PREHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year PREHOOK: type: QUERY PREHOOK: Input: default@loc +PREHOOK: Output: default@loc #### A masked pattern was here #### POSTHOOK: query: analyze table loc compute statistics for columns state,locid,zip,year POSTHOOK: type: QUERY POSTHOOK: Input: default@loc +POSTHOOK: Output: default@loc #### A masked pattern was here #### PREHOOK: query: explain select * from emp e join dept d on (e.deptid = d.deptid) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/auto_join1.q.out ql/src/test/results/clientpositive/spark/auto_join1.q.out index d9cd7700cc..49795dc71c 100644 --- ql/src/test/results/clientpositive/spark/auto_join1.q.out +++ ql/src/test/results/clientpositive/spark/auto_join1.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/auto_join14.q.out ql/src/test/results/clientpositive/spark/auto_join14.q.out index 82deefea73..356cc85703 100644 --- ql/src/test/results/clientpositive/spark/auto_join14.q.out +++ ql/src/test/results/clientpositive/spark/auto_join14.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/spark/auto_join17.q.out ql/src/test/results/clientpositive/spark/auto_join17.q.out index 6d63fa68a3..b3e01f2a08 100644 --- ql/src/test/results/clientpositive/spark/auto_join17.q.out +++ ql/src/test/results/clientpositive/spark/auto_join17.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.*, src2.* diff --git ql/src/test/results/clientpositive/spark/auto_join19.q.out ql/src/test/results/clientpositive/spark/auto_join19.q.out index 88ef3f1981..75426c15c1 100644 --- ql/src/test/results/clientpositive/spark/auto_join19.q.out +++ ql/src/test/results/clientpositive/spark/auto_join19.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/auto_join2.q.out ql/src/test/results/clientpositive/spark/auto_join2.q.out index e32abba2f4..126acada5b 100644 --- ql/src/test/results/clientpositive/spark/auto_join2.q.out +++ ql/src/test/results/clientpositive/spark/auto_join2.q.out @@ -124,7 +124,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/auto_join26.q.out ql/src/test/results/clientpositive/spark/auto_join26.q.out index bfb3564cf8..33e160993f 100644 --- ql/src/test/results/clientpositive/spark/auto_join26.q.out +++ ql/src/test/results/clientpositive/spark/auto_join26.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/spark/auto_join3.q.out ql/src/test/results/clientpositive/spark/auto_join3.q.out index a17cc1a0a1..11f909d4b8 100644 --- ql/src/test/results/clientpositive/spark/auto_join3.q.out +++ ql/src/test/results/clientpositive/spark/auto_join3.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/auto_join4.q.out ql/src/test/results/clientpositive/spark/auto_join4.q.out index a2c9a74cd9..0502e62354 100644 --- ql/src/test/results/clientpositive/spark/auto_join4.q.out +++ ql/src/test/results/clientpositive/spark/auto_join4.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join5.q.out ql/src/test/results/clientpositive/spark/auto_join5.q.out index 8adbf1cc8e..ae83ebc59c 100644 --- ql/src/test/results/clientpositive/spark/auto_join5.q.out +++ ql/src/test/results/clientpositive/spark/auto_join5.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join6.q.out ql/src/test/results/clientpositive/spark/auto_join6.q.out index 69df6cf684..b1ebb16e54 100644 --- ql/src/test/results/clientpositive/spark/auto_join6.q.out +++ ql/src/test/results/clientpositive/spark/auto_join6.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join7.q.out ql/src/test/results/clientpositive/spark/auto_join7.q.out index e02ae6f3a6..b4257507b6 100644 --- ql/src/test/results/clientpositive/spark/auto_join7.q.out +++ ql/src/test/results/clientpositive/spark/auto_join7.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join8.q.out ql/src/test/results/clientpositive/spark/auto_join8.q.out index 179e1d5445..a789e8cc0f 100644 --- ql/src/test/results/clientpositive/spark/auto_join8.q.out +++ ql/src/test/results/clientpositive/spark/auto_join8.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/auto_join9.q.out ql/src/test/results/clientpositive/spark/auto_join9.q.out index b2ed51c108..671eafe0b3 100644 --- ql/src/test/results/clientpositive/spark/auto_join9.q.out +++ ql/src/test/results/clientpositive/spark/auto_join9.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src2.value where src1.ds = '2008-04-08' and src1.hr = '12' diff --git ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out index c26949f13c..9308d476e3 100644 --- ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out +++ ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out @@ -1391,7 +1391,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1404,7 +1405,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key @@ -1611,7 +1613,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1624,7 +1627,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( select a.key as key, a.value as val1, b.value as val2 from tbl1 a join tbl2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out index e6038b857d..f67d967cc9 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_1.q.out @@ -169,10 +169,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -218,10 +216,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -377,10 +373,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -426,10 +420,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -575,10 +567,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -680,10 +670,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -729,10 +717,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out index 09caf944d2..b60da2e00a 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_12.q.out @@ -191,10 +191,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -267,10 +265,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -336,10 +332,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_medium numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_medium { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -448,10 +442,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -497,10 +489,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out index cd4b83a2a1..ded68d5e72 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_13.q.out @@ -134,7 +134,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -147,7 +148,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -319,7 +321,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -332,7 +335,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 @@ -532,7 +536,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -545,7 +550,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( SELECT a.key key1, a.value value1, b.key key2, b.value value2 diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out index 025d0d29c5..9cefd39b5d 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_2.q.out @@ -151,10 +151,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -200,10 +198,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -349,10 +345,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -454,10 +448,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -503,10 +495,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out index 3ad950a107..2e15cf0147 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_3.q.out @@ -151,10 +151,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -309,10 +307,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -457,10 +453,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -506,10 +500,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -612,10 +604,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out index 60437ec56d..e74df7da36 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_4.q.out @@ -167,10 +167,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -325,10 +323,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -473,10 +469,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -522,10 +516,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -628,10 +620,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out index 54f11c4302..b8b5fe360a 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_5.q.out @@ -81,16 +81,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -98,7 +98,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Position of Big Table: 1 - Statistics: Num rows: 1 Data size: 248 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Group By Operator aggregations: count() @@ -131,8 +131,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -153,8 +151,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -232,16 +228,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Sorted Merge Bucket Map Join Operator condition map: Inner Join 0 to 1 @@ -249,7 +245,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col0 (type: string) Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Group By Operator aggregations: count() @@ -282,8 +278,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -304,8 +298,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -382,16 +374,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 226 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 226 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 _col0 (type: string) @@ -423,8 +415,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -445,8 +435,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -468,16 +456,16 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 27 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -487,7 +475,7 @@ STAGE PLANS: input vertices: 1 Map 3 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 3025 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Group By Operator aggregations: count() @@ -527,8 +515,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -549,8 +535,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out index 16ecabe05d..e50fc62dc7 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_7.q.out @@ -184,10 +184,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -233,10 +231,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -394,10 +390,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -443,10 +437,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -594,10 +586,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -643,10 +633,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -749,10 +737,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -798,10 +784,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out index e180471dcb..7a49c4880b 100644 --- ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out +++ ql/src/test/results/clientpositive/spark/auto_sortmerge_join_8.q.out @@ -184,10 +184,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -233,10 +231,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -394,10 +390,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -443,10 +437,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -594,10 +586,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -643,10 +633,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_small numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_small { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -749,10 +737,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -798,10 +784,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.bucket_big numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct bucket_big { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out index 2a00cb32ec..16666c9277 100644 --- ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out +++ ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out @@ -25,10 +25,12 @@ POSTHOOK: Output: default@dec PREHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value PREHOOK: type: QUERY PREHOOK: Input: default@dec +PREHOOK: Output: default@dec #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `dec` COMPUTE STATISTICS FOR COLUMNS value POSTHOOK: type: QUERY POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec #### A masked pattern was here #### PREHOOK: query: DESC FORMATTED `dec` value PREHOOK: type: DESCTABLE @@ -48,7 +50,7 @@ num_trues num_falses bitVector comment from deserializer -COLUMN_STATS_ACCURATE {\"COLUMN_STATS\":{\"value\":\"true\"}} +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"value\":\"true\"}} PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/spark/bucket2.q.out ql/src/test/results/clientpositive/spark/bucket2.q.out index 90c9e5469d..32f12582af 100644 --- ql/src/test/results/clientpositive/spark/bucket2.q.out +++ ql/src/test/results/clientpositive/spark/bucket2.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/spark/bucket3.q.out ql/src/test/results/clientpositive/spark/bucket3.q.out index 078460f9b9..b88e73fab8 100644 --- ql/src/test/results/clientpositive/spark/bucket3.q.out +++ ql/src/test/results/clientpositive/spark/bucket3.q.out @@ -163,7 +163,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') diff --git ql/src/test/results/clientpositive/spark/bucket4.q.out ql/src/test/results/clientpositive/spark/bucket4.q.out index 13e21b6610..63683fc8bd 100644 --- ql/src/test/results/clientpositive/spark/bucket4.q.out +++ ql/src/test/results/clientpositive/spark/bucket4.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.bucket4_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket4_1 diff --git ql/src/test/results/clientpositive/spark/bucket5.q.out ql/src/test/results/clientpositive/spark/bucket5.q.out index 8aa4e5f6c5..e3e18a52d4 100644 --- ql/src/test/results/clientpositive/spark/bucket5.q.out +++ ql/src/test/results/clientpositive/spark/bucket5.q.out @@ -291,7 +291,8 @@ STAGE PLANS: name: default.bucketed_table Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -323,7 +324,8 @@ STAGE PLANS: name: default.unbucketed_table Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM src diff --git ql/src/test/results/clientpositive/spark/bucket6.q.out ql/src/test/results/clientpositive/spark/bucket6.q.out index d5d53d303d..b5d33471eb 100644 --- ql/src/test/results/clientpositive/spark/bucket6.q.out +++ ql/src/test/results/clientpositive/spark/bucket6.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.src_bucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table src_bucket select key,value from srcpart PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out index b57ba19e04..37c7715494 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_1.q.out @@ -94,8 +94,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table2 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -116,8 +114,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table2 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -188,8 +184,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -210,8 +204,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out index 4b8f98581b..f09f34ae14 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_2.q.out @@ -94,8 +94,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table2 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -116,8 +114,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table2 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -188,8 +184,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -210,8 +204,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.table1 numFiles 1 - numRows 0 - rawDataSize 0 serialization.ddl struct table1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out index 194f16ed9c..ad148d0bdb 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_spark1.q.out @@ -173,10 +173,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -300,10 +298,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -363,7 +359,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -475,10 +472,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -596,10 +591,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -659,7 +652,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out index bb66d1ec47..37b80f064d 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_spark2.q.out @@ -157,10 +157,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -284,10 +282,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -347,7 +343,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -464,10 +461,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -591,10 +586,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -654,7 +647,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out index 10678a4c26..29e9a839b7 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_spark3.q.out @@ -157,10 +157,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -284,10 +282,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -347,7 +343,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -459,10 +456,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -580,10 +575,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -643,7 +636,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out index c0197d32fc..c9597ab503 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out @@ -112,41 +112,57 @@ PREHOOK: query: analyze table srcbucket_mapjoin compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: Output: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table srcbucket_mapjoin compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: analyze table srcbucket_mapjoin_part compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_part PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table srcbucket_mapjoin_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_part POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: analyze table tab compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tab PREHOOK: Input: default@tab@ds=2008-04-08 +PREHOOK: Output: default@tab +PREHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab POSTHOOK: Input: default@tab@ds=2008-04-08 +POSTHOOK: Output: default@tab +POSTHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: analyze table tab_part compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tab_part PREHOOK: Input: default@tab_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part +PREHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part +POSTHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: explain select a.key, a.value, b.value @@ -2248,25 +2264,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), substr(value, 5) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col1) keys: _col0 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Map 4 Map Operator Tree: @@ -2293,16 +2309,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: double) Reducer 3 Reduce Operator Tree: @@ -2358,25 +2374,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), substr(value, 5) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col1) keys: _col0 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Map 4 Map Operator Tree: @@ -2403,16 +2419,16 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: double), _col0 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: double) Reducer 3 Reduce Operator Tree: @@ -3494,25 +3510,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), substr(value, 5) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col1) keys: _col0 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Map 4 Map Operator Tree: @@ -3539,12 +3555,12 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Reducer 3 Reduce Operator Tree: @@ -3600,25 +3616,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), substr(value, 5) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col1) keys: _col0 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Map 4 Map Operator Tree: @@ -3645,12 +3661,12 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Reducer 3 Reduce Operator Tree: @@ -3706,19 +3722,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), substr(value, 5) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -3745,12 +3761,12 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Reducer 3 Reduce Operator Tree: @@ -3806,19 +3822,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), substr(value, 5) (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 27 Data size: 2808 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Map 4 Map Operator Tree: @@ -3845,12 +3861,12 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: complete outputColumnNames: _col0, _col1 - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: double) Reducer 3 Reduce Operator Tree: diff --git ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out index 34837f52dd..294f50c5e8 100644 --- ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out +++ ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out @@ -112,41 +112,57 @@ PREHOOK: query: analyze table srcbucket_mapjoin compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: Output: default@srcbucket_mapjoin +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table srcbucket_mapjoin compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: analyze table srcbucket_mapjoin_part compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_part PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table srcbucket_mapjoin_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_part POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: analyze table tab compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tab PREHOOK: Input: default@tab@ds=2008-04-08 +PREHOOK: Output: default@tab +PREHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab POSTHOOK: Input: default@tab@ds=2008-04-08 +POSTHOOK: Output: default@tab +POSTHOOK: Output: default@tab@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: analyze table tab_part compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tab_part PREHOOK: Input: default@tab_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part +PREHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_part POSTHOOK: Input: default@tab_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part +POSTHOOK: Output: default@tab_part@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: explain select a.key, b.key from tab_part a join tab_part c on a.key = c.key join tab_part b on a.value = b.value PREHOOK: type: QUERY @@ -399,10 +415,12 @@ POSTHOOK: Lineage: tab1.value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.Field PREHOOK: query: analyze table tab1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tab1 +PREHOOK: Output: default@tab1 #### A masked pattern was here #### POSTHOOK: query: analyze table tab1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab1 +POSTHOOK: Output: default@tab1 #### A masked pattern was here #### PREHOOK: query: explain select a.key, a.value, b.value @@ -1406,11 +1424,15 @@ PREHOOK: query: analyze table tab_part1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@tab_part1 PREHOOK: Input: default@tab_part1@ds=2008-04-08 +PREHOOK: Output: default@tab_part1 +PREHOOK: Output: default@tab_part1@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: analyze table tab_part1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_part1 POSTHOOK: Input: default@tab_part1@ds=2008-04-08 +POSTHOOK: Output: default@tab_part1 +POSTHOOK: Output: default@tab_part1@ds=2008-04-08 #### A masked pattern was here #### PREHOOK: query: explain select count(*) diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out index 0b1ee92a48..521ced46b2 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin1.q.out @@ -390,22 +390,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: false @@ -427,8 +427,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -448,8 +446,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -503,10 +499,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -613,7 +607,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -745,22 +740,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: false @@ -782,8 +777,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -803,8 +796,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -858,10 +849,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -968,7 +957,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out index 9664fb8ee9..5a41948e88 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin10.q.out @@ -183,10 +183,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 3 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -231,10 +229,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -326,10 +322,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -374,10 +368,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 3 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out index 00161e3356..b81601353e 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin11.q.out @@ -196,10 +196,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -244,10 +242,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -345,10 +341,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -393,10 +387,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -549,10 +541,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -597,10 +587,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -698,10 +686,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -746,10 +732,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 4 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out index 1c26614b20..0936a5e660 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin12.q.out @@ -157,10 +157,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -256,10 +254,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -401,10 +397,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_3 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_3 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -495,10 +489,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out index 49d7cc9928..3eca1e5cfa 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin2.q.out @@ -155,10 +155,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -231,10 +229,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -341,7 +337,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -516,10 +513,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -592,10 +587,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -702,7 +695,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -894,10 +888,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -970,10 +962,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1018,10 +1008,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1129,7 +1117,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out index 1db5b2c142..f0a42b8419 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin3.q.out @@ -179,10 +179,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -255,10 +253,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -365,7 +361,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -540,10 +537,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -616,10 +611,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -726,7 +719,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out index 563e816788..4f3632a09d 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin4.q.out @@ -140,22 +140,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: false @@ -177,8 +177,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -198,8 +196,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -214,22 +210,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string) auto parallelism: false @@ -251,8 +247,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -272,8 +266,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -294,17 +286,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToString(_col0) (type: string), _col1 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -361,7 +353,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -485,22 +478,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 0 value expressions: _col1 (type: string) auto parallelism: false @@ -522,8 +515,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -543,8 +534,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -559,22 +548,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: b - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE tag: 1 value expressions: _col1 (type: string) auto parallelism: false @@ -596,8 +585,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -617,8 +604,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -639,17 +624,17 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: UDFToString(_col0) (type: string), _col1 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 3025 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 3025 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -706,7 +691,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out index 5e5761813d..9fb859fcf8 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin5.q.out @@ -189,12 +189,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: int) @@ -225,8 +225,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -246,8 +244,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -348,10 +344,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -396,10 +390,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 4 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -460,7 +452,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result @@ -595,12 +588,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: int) @@ -631,8 +624,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -652,8 +643,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -754,10 +743,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -802,10 +789,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -866,7 +851,8 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out index bcf5b79dfd..0d7def7b07 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out @@ -117,10 +117,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -220,10 +218,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns ds/hr partition_columns.types string:string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out index 5cf7a7b31d..7f6eb9a00d 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin8.q.out @@ -122,10 +122,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -222,10 +220,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -381,10 +377,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -481,10 +475,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out index 2e3982173d..d72279f072 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin9.q.out @@ -125,10 +125,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 3 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -219,10 +217,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -406,10 +402,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -500,10 +494,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_1 numFiles 2 - numRows 0 partition_columns part partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out index 2bfb4afea6..937770b421 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin_negative.q.out @@ -123,10 +123,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part numFiles 3 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -165,12 +163,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -238,8 +236,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -259,8 +255,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -301,6 +295,7 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out index 277eb7fb51..b84e9eafbb 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin_negative2.q.out @@ -137,10 +137,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -185,10 +183,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin_part_2 numFiles 2 - numRows 0 partition_columns ds partition_columns.types string - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -228,12 +224,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: a - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: key is not null (type: boolean) - Statistics: Num rows: 1 Data size: 2750 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 26 Data size: 2750 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -307,8 +303,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -328,8 +322,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.srcbucket_mapjoin numFiles 2 - numRows 0 - rawDataSize 0 serialization.ddl struct srcbucket_mapjoin { i32 key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -370,6 +362,7 @@ STAGE PLANS: name: default.bucketmapjoin_tmp_result Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out index 2fa0214f20..cd6cf633e6 100644 --- ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out +++ ql/src/test/results/clientpositive/spark/bucketmapjoin_negative3.q.out @@ -160,12 +160,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -197,8 +197,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -219,8 +217,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -240,12 +236,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -256,18 +252,18 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -310,8 +306,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -332,8 +326,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -369,12 +361,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -406,8 +398,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -428,8 +418,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -449,12 +437,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -465,18 +453,18 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE BucketMapJoin: true Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -519,8 +507,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -541,8 +527,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -578,12 +562,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: UDFToDouble(key) is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 (key + key) (type: double) @@ -610,8 +594,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -632,8 +614,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -653,12 +633,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key + key) is not null (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -669,17 +649,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -717,8 +697,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -739,8 +717,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -776,12 +752,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -808,8 +784,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -830,8 +804,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -851,12 +823,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -867,17 +839,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -915,8 +887,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -937,8 +907,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -974,12 +942,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1006,8 +974,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1028,8 +994,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1049,12 +1013,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1065,17 +1029,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1113,8 +1077,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1135,8 +1097,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1172,12 +1132,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1204,8 +1164,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test4 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1226,8 +1184,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test4 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1247,12 +1203,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1263,17 +1219,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1311,8 +1267,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1333,8 +1287,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test1 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test1 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1370,12 +1322,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1402,8 +1354,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1424,8 +1374,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1445,12 +1393,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1461,17 +1409,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1509,8 +1457,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1531,8 +1477,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1568,12 +1512,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1600,8 +1544,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test4 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1622,8 +1564,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test4 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1643,12 +1583,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1659,17 +1599,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1707,8 +1647,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1729,8 +1667,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test2 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test2 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1766,12 +1702,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: r - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: 0 key (type: string), value (type: string) @@ -1798,8 +1734,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test4 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1820,8 +1754,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test4 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test4 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1841,12 +1773,12 @@ STAGE PLANS: Map Operator Tree: TableScan alias: l - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key is not null and value is not null) (type: boolean) - Statistics: Num rows: 1 Data size: 4200 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 21 Data size: 4200 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Inner Join 0 to 1 @@ -1857,17 +1789,17 @@ STAGE PLANS: input vertices: 1 Map 2 Position of Big Table: 0 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 4620 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 23 Data size: 4620 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -1905,8 +1837,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -1927,8 +1857,6 @@ STAGE PLANS: #### A masked pattern was here #### name default.test3 numFiles 3 - numRows 0 - rawDataSize 0 serialization.ddl struct test3 { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out index 81a064b2b7..814553d81a 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out @@ -186,7 +186,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -362,7 +363,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -562,7 +564,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -768,7 +771,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -956,7 +960,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.v1, b.v2) @@ -1144,7 +1149,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key+a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out index 4245aa1d99..faada6fff0 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key, concat(a.value, b.value) @@ -345,7 +346,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.value diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out index 5e4e5ef8ad..8282a5a33f 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key2, concat(a.value, b.value) @@ -347,7 +348,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq1.key, subq1.key2, subq1.value from @@ -530,7 +532,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -648,7 +651,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') @@ -772,7 +776,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.key, subq2.key2, subq2.value from @@ -973,7 +978,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT subq2.k2, subq2.k1, subq2.value from @@ -1184,5 +1190,6 @@ STAGE PLANS: name: default.test_table4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out index 52bf6eeee7..c9b03a1283 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -344,7 +345,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) @@ -530,7 +532,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out index 8263b6f681..2198dc2fa2 100644 --- ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out +++ ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out @@ -162,7 +162,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, b.key, concat(a.value, b.value) @@ -339,7 +340,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT b.key, a.key, concat(a.value, b.value) diff --git ql/src/test/results/clientpositive/spark/ctas.q.out ql/src/test/results/clientpositive/spark/ctas.q.out index d958a32fd8..2ccd1ddfcf 100644 --- ql/src/test/results/clientpositive/spark/ctas.q.out +++ ql/src/test/results/clientpositive/spark/ctas.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.nzhang_CTAS1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_CTAS1 as select key k, value from src sort by k, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -240,7 +241,8 @@ STAGE PLANS: name: default.nzhang_ctas2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas2 as select * from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -386,7 +388,8 @@ STAGE PLANS: name: default.nzhang_ctas3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas3 row format serde "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe" stored as RCFile as select key/2 half_key, concat(value, "_con") conb from src sort by half_key, conb limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -596,7 +599,8 @@ STAGE PLANS: name: default.nzhang_ctas4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas4 row format delimited fields terminated by ',' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT @@ -746,7 +750,8 @@ STAGE PLANS: name: default.nzhang_ctas5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table nzhang_ctas5 row format delimited fields terminated by ',' lines terminated by '\012' stored as textfile as select key, value from src sort by key, value limit 10 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out index 8693026c71..86d6e1a874 100644 --- ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out +++ ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out @@ -166,7 +166,8 @@ STAGE PLANS: name: default.bucket2_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket2_1 diff --git ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out index 4f4dddd10d..0379c9ce80 100644 --- ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out +++ ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out @@ -239,7 +239,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -252,7 +253,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: SELECT dest1.* FROM dest1 PREHOOK: type: QUERY @@ -453,7 +455,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: SELECT * FROM tmptable x SORT BY x.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1.q.out ql/src/test/results/clientpositive/spark/groupby1.q.out index 42ce2430d3..68cb6d7176 100644 --- ql/src/test/results/clientpositive/spark/groupby1.q.out +++ ql/src/test/results/clientpositive/spark/groupby1.q.out @@ -86,7 +86,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby10.q.out ql/src/test/results/clientpositive/spark/groupby10.q.out index b7d19fe1af..f03e4e3b70 100644 --- ql/src/test/results/clientpositive/spark/groupby10.q.out +++ ql/src/test/results/clientpositive/spark/groupby10.q.out @@ -151,7 +151,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -164,7 +165,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -363,7 +365,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -376,7 +379,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, count(substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key @@ -544,7 +548,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -557,7 +562,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM INPUT INSERT OVERWRITE TABLE dest1 SELECT INPUT.key, sum(distinct substr(INPUT.value,5)), count(distinct substr(INPUT.value,5)) GROUP BY INPUT.key diff --git ql/src/test/results/clientpositive/spark/groupby11.q.out ql/src/test/results/clientpositive/spark/groupby11.q.out index a0f99c4cfa..94ab3e0c44 100644 --- ql/src/test/results/clientpositive/spark/groupby11.q.out +++ ql/src/test/results/clientpositive/spark/groupby11.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -170,7 +171,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 partition(ds='111') diff --git ql/src/test/results/clientpositive/spark/groupby1_map.q.out ql/src/test/results/clientpositive/spark/groupby1_map.q.out index b414aa62a3..775ca8a6b8 100644 --- ql/src/test/results/clientpositive/spark/groupby1_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_map.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out index a01cee1d49..20e9c6566e 100644 --- ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_map_nomap.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out index f7b7f7a185..8c3f571830 100644 --- ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_map_skew.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out index 1b7e53b3eb..ea607cdf82 100644 --- ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby1_noskew.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest_g1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,5)) GROUP BY src.key PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby2.q.out ql/src/test/results/clientpositive/spark/groupby2.q.out index a5cd0e6066..966c273235 100644 --- ql/src/test/results/clientpositive/spark/groupby2.q.out +++ ql/src/test/results/clientpositive/spark/groupby2.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_map.q.out ql/src/test/results/clientpositive/spark/groupby2_map.q.out index d2b69af851..c716bbd579 100644 --- ql/src/test/results/clientpositive/spark/groupby2_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_map.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out index 4ad056887d..faf69b68f6 100644 --- ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_map_multi_distinct.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) @@ -187,7 +188,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.key,1,1)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out index f4a567ea9b..e2974249aa 100644 --- ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_map_skew.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out index 8ecf769b43..83e794a9c1 100644 --- ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_noskew.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out index 3ede0fc755..171932ad1b 100644 --- ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby2_noskew_multi_distinct.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby3.q.out ql/src/test/results/clientpositive/spark/groupby3.q.out index a34e89e1ef..c314763db2 100644 --- ql/src/test/results/clientpositive/spark/groupby3.q.out +++ ql/src/test/results/clientpositive/spark/groupby3.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_map.q.out ql/src/test/results/clientpositive/spark/groupby3_map.q.out index 71f8dc0191..9caf47b676 100644 --- ql/src/test/results/clientpositive/spark/groupby3_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_map.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out index 47ef5cb6ae..678276115f 100644 --- ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_map_multi_distinct.q.out @@ -99,7 +99,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out index 7cfca81d71..2f101601fe 100644 --- ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_map_skew.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out index b2993a6e85..d07eea5fe6 100644 --- ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_noskew.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out index d152a07c77..8bcb912991 100644 --- ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby3_noskew_multi_distinct.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT diff --git ql/src/test/results/clientpositive/spark/groupby4.q.out ql/src/test/results/clientpositive/spark/groupby4.q.out index 3ad01d0d70..1146c8404d 100644 --- ql/src/test/results/clientpositive/spark/groupby4.q.out +++ ql/src/test/results/clientpositive/spark/groupby4.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby4_map.q.out ql/src/test/results/clientpositive/spark/groupby4_map.q.out index 7cb360033e..96f34883ac 100644 --- ql/src/test/results/clientpositive/spark/groupby4_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby4_map.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out index ef287ad05a..3bf8d9985e 100644 --- ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby4_map_skew.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out index 04f58fa671..4af92e8b94 100644 --- ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby4_noskew.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,1,1) GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby5.q.out ql/src/test/results/clientpositive/spark/groupby5.q.out index d292f747e6..8dfed8be31 100644 --- ql/src/test/results/clientpositive/spark/groupby5.q.out +++ ql/src/test/results/clientpositive/spark/groupby5.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/spark/groupby5_map.q.out ql/src/test/results/clientpositive/spark/groupby5_map.q.out index add30941b2..96cf6b7fd6 100644 --- ql/src/test/results/clientpositive/spark/groupby5_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby5_map.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out index 924ef5dbc7..11a65580ab 100644 --- ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby5_map_skew.q.out @@ -73,7 +73,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out index 300ccb64c3..16ac57bbfa 100644 --- ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby5_noskew.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,5)) diff --git ql/src/test/results/clientpositive/spark/groupby6.q.out ql/src/test/results/clientpositive/spark/groupby6.q.out index 4f406d7d07..04e6127413 100644 --- ql/src/test/results/clientpositive/spark/groupby6.q.out +++ ql/src/test/results/clientpositive/spark/groupby6.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby6_map.q.out ql/src/test/results/clientpositive/spark/groupby6_map.q.out index 03f68c63b2..509014a2b7 100644 --- ql/src/test/results/clientpositive/spark/groupby6_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby6_map.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out index 606b5d5272..e2260eda2b 100644 --- ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby6_map_skew.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out index eb72f011b2..f95551942c 100644 --- ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby6_noskew.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,5,1) diff --git ql/src/test/results/clientpositive/spark/groupby7_map.q.out ql/src/test/results/clientpositive/spark/groupby7_map.q.out index 31daab8acc..9260e7d408 100644 --- ql/src/test/results/clientpositive/spark/groupby7_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_map.q.out @@ -112,7 +112,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -125,7 +126,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out index 625a7374c9..c628b3932b 100644 --- ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_map_multi_single_reducer.q.out @@ -105,7 +105,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -118,7 +119,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out index 4fbfd30ec6..e55a79d40b 100644 --- ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_map_skew.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -140,7 +141,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out index a26247af8d..9e05b99ac9 100644 --- ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_noskew.q.out @@ -106,7 +106,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -119,7 +120,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out index 2dce3016b3..9a9f7b28aa 100644 --- ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby7_noskew_multi_single_reducer.q.out @@ -125,7 +125,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -138,7 +139,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,5)) GROUP BY SRC.key ORDER BY SRC.key limit 10 diff --git ql/src/test/results/clientpositive/spark/groupby8.q.out ql/src/test/results/clientpositive/spark/groupby8.q.out index f7f3279e6b..c66a53dda7 100644 --- ql/src/test/results/clientpositive/spark/groupby8.q.out +++ ql/src/test/results/clientpositive/spark/groupby8.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -133,7 +134,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -893,7 +895,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -906,7 +909,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby8_map.q.out ql/src/test/results/clientpositive/spark/groupby8_map.q.out index 288ca3f3b2..25c07d0ba6 100644 --- ql/src/test/results/clientpositive/spark/groupby8_map.q.out +++ ql/src/test/results/clientpositive/spark/groupby8_map.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -117,7 +118,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out index 9e76fd57aa..1da8890949 100644 --- ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out +++ ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out @@ -126,7 +126,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -139,7 +140,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out index 288ca3f3b2..25c07d0ba6 100644 --- ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out +++ ql/src/test/results/clientpositive/spark/groupby8_noskew.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -117,7 +118,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby9.q.out ql/src/test/results/clientpositive/spark/groupby9.q.out index d59d8cf706..c016e4b6e2 100644 --- ql/src/test/results/clientpositive/spark/groupby9.q.out +++ ql/src/test/results/clientpositive/spark/groupby9.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -144,7 +145,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -916,7 +918,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -929,7 +932,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -1701,7 +1705,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1714,7 +1719,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -2488,7 +2494,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2501,7 +2508,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(SUBSTR(SRC.value,5)) GROUP BY SRC.key @@ -3273,7 +3281,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3286,7 +3295,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key diff --git ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out index 169ee04403..3b5ec42472 100644 --- ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out +++ ql/src/test/results/clientpositive/spark/groupby_complex_types.q.out @@ -175,7 +175,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -188,7 +189,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -201,7 +203,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key), COUNT(1) GROUP BY ARRAY(SRC.key) diff --git ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out index 7a730d230d..71e74a92ae 100644 --- ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby_complex_types_multi_single_reducer.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -172,7 +173,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT ARRAY(SRC.key) as keyarray, COUNT(1) GROUP BY ARRAY(SRC.key) ORDER BY keyarray limit 10 diff --git ql/src/test/results/clientpositive/spark/groupby_cube1.q.out ql/src/test/results/clientpositive/spark/groupby_cube1.q.out index 52c87efaf0..7629fe6620 100644 --- ql/src/test/results/clientpositive/spark/groupby_cube1.q.out +++ ql/src/test/results/clientpositive/spark/groupby_cube1.q.out @@ -704,7 +704,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -717,7 +718,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with cube diff --git ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out index f3a95bdde5..d1fcd48084 100644 --- ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out +++ ql/src/test/results/clientpositive/spark/groupby_map_ppr.q.out @@ -234,7 +234,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out index 666f355a4c..30e5daa437 100644 --- ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby_map_ppr_multi_distinct.q.out @@ -234,7 +234,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out index bf4132a974..f88ff7ae9f 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out @@ -131,7 +131,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -144,7 +145,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table dest1 select key, count(distinct value) group by key diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out index c16df1b667..89db8ce177 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -170,7 +171,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -183,7 +185,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) @@ -468,7 +471,8 @@ STAGE PLANS: name: default.dest_g4 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -481,7 +485,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -494,7 +499,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -507,7 +513,8 @@ STAGE PLANS: name: default.dest_h2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Move Operator @@ -520,7 +527,8 @@ STAGE PLANS: name: default.dest_h3 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT substr(src.value,5)), concat(substr(src.key,1,1),sum(substr(src.value,5))), sum(DISTINCT substr(src.value, 5)), count(src.value) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out index 37deb9336f..a90082f341 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer2.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest_g2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -123,7 +124,8 @@ STAGE PLANS: name: default.dest_g3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest_g2 SELECT substr(src.key,1,1), count(DISTINCT src.key) WHERE substr(src.key,1,1) >= 5 GROUP BY substr(src.key,1,1) diff --git ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out index f9eaa3c116..50ae717839 100644 --- ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out +++ ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out @@ -122,7 +122,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -135,7 +136,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -294,7 +296,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -307,7 +310,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -466,7 +470,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -479,7 +484,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 @@ -638,7 +644,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -651,7 +658,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table e1 diff --git ql/src/test/results/clientpositive/spark/groupby_position.q.out ql/src/test/results/clientpositive/spark/groupby_position.q.out index 91c2581967..a88176fa6b 100644 --- ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -142,7 +143,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 @@ -320,7 +322,8 @@ STAGE PLANS: name: default.testtable1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -333,7 +336,8 @@ STAGE PLANS: name: default.testtable2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM SRC INSERT OVERWRITE TABLE testTable1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) WHERE SRC.key < 20 GROUP BY 1 diff --git ql/src/test/results/clientpositive/spark/groupby_ppr.q.out ql/src/test/results/clientpositive/spark/groupby_ppr.q.out index e45b5c2030..2bbffaf77e 100644 --- ql/src/test/results/clientpositive/spark/groupby_ppr.q.out +++ ql/src/test/results/clientpositive/spark/groupby_ppr.q.out @@ -227,7 +227,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out index c3c97f5a10..14c287f13d 100644 --- ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out +++ ql/src/test/results/clientpositive/spark/groupby_ppr_multi_distinct.q.out @@ -227,7 +227,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src @@ -494,7 +495,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src diff --git ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out index 68670ab2fc..29b111fc51 100644 --- ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out +++ ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out @@ -537,7 +537,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -550,7 +551,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T1 INSERT OVERWRITE TABLE T2 SELECT key, val, count(1) group by key, val with rollup diff --git ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 8292e3a6b4..5ceb9e9dd9 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -189,7 +189,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -400,7 +401,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -585,7 +587,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -768,7 +771,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -959,7 +963,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1171,7 +1176,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -1377,7 +1383,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1595,7 +1602,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -1897,7 +1905,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2232,7 +2241,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2538,7 +2548,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3014,7 +3025,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3197,7 +3209,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3391,7 +3404,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl5 @@ -3584,7 +3598,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3784,7 +3799,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3941,7 +3957,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3954,7 +3971,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -4105,7 +4123,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4118,7 +4137,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 42847269f3..4b11eb48bc 100644 --- ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -189,7 +189,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -419,7 +420,8 @@ STAGE PLANS: name: default.outputtbl2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl2 @@ -604,7 +606,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -787,7 +790,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -978,7 +982,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1209,7 +1214,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -1434,7 +1440,8 @@ STAGE PLANS: name: default.outputtbl3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl3 @@ -1671,7 +1678,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -1973,7 +1981,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2327,7 +2336,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -2633,7 +2643,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3147,7 +3158,8 @@ STAGE PLANS: name: default.outputtbl1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl1 @@ -3330,7 +3342,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3524,7 +3537,8 @@ STAGE PLANS: name: default.outputtbl5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl5 @@ -3717,7 +3731,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -3917,7 +3932,8 @@ STAGE PLANS: name: default.outputtbl4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE outputTbl4 @@ -4089,7 +4105,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4102,7 +4119,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM T2 INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key @@ -4268,7 +4286,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -4281,7 +4300,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, val from T2 where key = 8) x INSERT OVERWRITE TABLE DEST1 SELECT key, count(1) GROUP BY key diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out index 49eafbe8c2..e0d5e58e2b 100644 --- ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out +++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_map_operators.q.out @@ -103,7 +103,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT key, count(*) FROM test_table1 GROUP BY key @@ -260,7 +261,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT a.key, a.value FROM ( @@ -381,7 +383,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT /*+ MAPJOIN(a) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key @@ -516,7 +519,8 @@ STAGE PLANS: name: default.test_table_out Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table_out PARTITION (part = '1') SELECT /*+ MAPJOIN(a) */ b.value, count(*) FROM test_table1 a JOIN test_table2 b ON a.key = b.key diff --git ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out index 8da2705738..1bdf64158b 100644 --- ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out +++ ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out @@ -77,7 +77,8 @@ STAGE PLANS: name: default.test_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table PARTITION (ds = '2008-04-08', hr) SELECT key2, value, cast(hr as int) FROM diff --git ql/src/test/results/clientpositive/spark/innerjoin.q.out ql/src/test/results/clientpositive/spark/innerjoin.q.out index 9328b99b0f..c02aa2a1e1 100644 --- ql/src/test/results/clientpositive/spark/innerjoin.q.out +++ ql/src/test/results/clientpositive/spark/innerjoin.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 INNER JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/input12.q.out ql/src/test/results/clientpositive/spark/input12.q.out index efbaaf9086..2fba863851 100644 --- ql/src/test/results/clientpositive/spark/input12.q.out +++ ql/src/test/results/clientpositive/spark/input12.q.out @@ -110,7 +110,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -123,7 +124,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -139,7 +141,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 diff --git ql/src/test/results/clientpositive/spark/input13.q.out ql/src/test/results/clientpositive/spark/input13.q.out index a72898fddb..dc5eec398a 100644 --- ql/src/test/results/clientpositive/spark/input13.q.out +++ ql/src/test/results/clientpositive/spark/input13.q.out @@ -127,7 +127,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -140,7 +141,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -156,7 +158,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator diff --git ql/src/test/results/clientpositive/spark/input14.q.out ql/src/test/results/clientpositive/spark/input14.q.out index 36f162eb90..ef2f37cf70 100644 --- ql/src/test/results/clientpositive/spark/input14.q.out +++ ql/src/test/results/clientpositive/spark/input14.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/spark/input17.q.out ql/src/test/results/clientpositive/spark/input17.q.out index d95dbcb61a..5f207aa58e 100644 --- ql/src/test/results/clientpositive/spark/input17.q.out +++ ql/src/test/results/clientpositive/spark/input17.q.out @@ -84,7 +84,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src_thrift diff --git ql/src/test/results/clientpositive/spark/input18.q.out ql/src/test/results/clientpositive/spark/input18.q.out index 65850b2ef5..ba77d156ec 100644 --- ql/src/test/results/clientpositive/spark/input18.q.out +++ ql/src/test/results/clientpositive/spark/input18.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM src diff --git ql/src/test/results/clientpositive/spark/input1_limit.q.out ql/src/test/results/clientpositive/spark/input1_limit.q.out index dd49287a2c..cfc96cd563 100644 --- ql/src/test/results/clientpositive/spark/input1_limit.q.out +++ ql/src/test/results/clientpositive/spark/input1_limit.q.out @@ -133,7 +133,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -146,7 +147,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value WHERE src.key < 100 LIMIT 10 diff --git ql/src/test/results/clientpositive/spark/input_part2.q.out ql/src/test/results/clientpositive/spark/input_part2.q.out index a728e977a5..12a0afbfa5 100644 --- ql/src/test/results/clientpositive/spark/input_part2.q.out +++ ql/src/test/results/clientpositive/spark/input_part2.q.out @@ -254,7 +254,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -286,7 +287,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart diff --git ql/src/test/results/clientpositive/spark/insert1.q.out ql/src/test/results/clientpositive/spark/insert1.q.out index fa8757f465..25f7289fae 100644 --- ql/src/test/results/clientpositive/spark/insert1.q.out +++ ql/src/test/results/clientpositive/spark/insert1.q.out @@ -70,7 +70,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY @@ -118,7 +119,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create database x PREHOOK: type: CREATEDATABASE @@ -180,7 +182,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain insert into table default.INSERT1 select a.key, a.value from insert2 a WHERE (a.key=-1) PREHOOK: type: QUERY @@ -228,7 +231,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain from insert2 @@ -299,7 +303,8 @@ STAGE PLANS: name: default.insert1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -312,7 +317,8 @@ STAGE PLANS: name: x.insert1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE DATABASE db2 PREHOOK: type: CREATEDATABASE diff --git ql/src/test/results/clientpositive/spark/insert_into1.q.out ql/src/test/results/clientpositive/spark/insert_into1.q.out index dff389db9c..4a5c3b804d 100644 --- ql/src/test/results/clientpositive/spark/insert_into1.q.out +++ ql/src/test/results/clientpositive/spark/insert_into1.q.out @@ -74,7 +74,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -188,7 +189,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 100 PREHOOK: type: QUERY @@ -302,7 +304,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * FROM src ORDER BY key LIMIT 10 PREHOOK: type: QUERY @@ -396,7 +399,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table insert_into1 select 1, 'a' PREHOOK: type: QUERY @@ -452,7 +456,8 @@ STAGE PLANS: name: default.insert_into1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into insert_into1 select 2, 'b' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/insert_into2.q.out ql/src/test/results/clientpositive/spark/insert_into2.q.out index 329387dd91..6724963896 100644 --- ql/src/test/results/clientpositive/spark/insert_into2.q.out +++ ql/src/test/results/clientpositive/spark/insert_into2.q.out @@ -80,7 +80,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT INTO TABLE insert_into2 PARTITION (ds='1') SELECT * FROM src order by key limit 100 PREHOOK: type: QUERY @@ -235,7 +236,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 100 @@ -359,7 +361,8 @@ STAGE PLANS: name: default.insert_into2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE insert_into2 PARTITION (ds='2') SELECT * FROM src order by key LIMIT 50 diff --git ql/src/test/results/clientpositive/spark/insert_into3.q.out ql/src/test/results/clientpositive/spark/insert_into3.q.out index a6fac2336b..a75cb81652 100644 --- ql/src/test/results/clientpositive/spark/insert_into3.q.out +++ ql/src/test/results/clientpositive/spark/insert_into3.q.out @@ -125,7 +125,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -138,7 +139,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT INTO TABLE insert_into3a SELECT * ORDER BY key, value LIMIT 50 INSERT INTO TABLE insert_into3b SELECT * ORDER BY key, value LIMIT 100 @@ -274,7 +276,8 @@ STAGE PLANS: name: default.insert_into3a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -287,7 +290,8 @@ STAGE PLANS: name: default.insert_into3b Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE insert_into3a SELECT * LIMIT 10 INSERT INTO TABLE insert_into3b SELECT * LIMIT 10 diff --git ql/src/test/results/clientpositive/spark/join1.q.out ql/src/test/results/clientpositive/spark/join1.q.out index a0ee4ea5b8..d89d3e5891 100644 --- ql/src/test/results/clientpositive/spark/join1.q.out +++ ql/src/test/results/clientpositive/spark/join1.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/join14.q.out ql/src/test/results/clientpositive/spark/join14.q.out index e804a1d9a5..11edde04be 100644 --- ql/src/test/results/clientpositive/spark/join14.q.out +++ ql/src/test/results/clientpositive/spark/join14.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src JOIN srcpart ON src.key = srcpart.key AND srcpart.ds = '2008-04-08' and src.key > 100 INSERT OVERWRITE TABLE dest1 SELECT src.key, srcpart.value diff --git ql/src/test/results/clientpositive/spark/join17.q.out ql/src/test/results/clientpositive/spark/join17.q.out index 3644efdde1..e8619feb0b 100644 --- ql/src/test/results/clientpositive/spark/join17.q.out +++ ql/src/test/results/clientpositive/spark/join17.q.out @@ -251,7 +251,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/spark/join2.q.out ql/src/test/results/clientpositive/spark/join2.q.out index f684beb7b9..d2d6b1d992 100644 --- ql/src/test/results/clientpositive/spark/join2.q.out +++ ql/src/test/results/clientpositive/spark/join2.q.out @@ -129,7 +129,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest_j2 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/join25.q.out ql/src/test/results/clientpositive/spark/join25.q.out index 05e5e701f9..0514466468 100644 --- ql/src/test/results/clientpositive/spark/join25.q.out +++ ql/src/test/results/clientpositive/spark/join25.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/spark/join26.q.out ql/src/test/results/clientpositive/spark/join26.q.out index ccd4526707..e8186abf24 100644 --- ql/src/test/results/clientpositive/spark/join26.q.out +++ ql/src/test/results/clientpositive/spark/join26.q.out @@ -329,7 +329,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join27.q.out ql/src/test/results/clientpositive/spark/join27.q.out index e10d2fb04a..40a91e7d22 100644 --- ql/src/test/results/clientpositive/spark/join27.q.out +++ ql/src/test/results/clientpositive/spark/join27.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/spark/join28.q.out ql/src/test/results/clientpositive/spark/join28.q.out index 8d4d870f89..4a3343adbf 100644 --- ql/src/test/results/clientpositive/spark/join28.q.out +++ ql/src/test/results/clientpositive/spark/join28.q.out @@ -136,7 +136,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq.key1, z.value diff --git ql/src/test/results/clientpositive/spark/join29.q.out ql/src/test/results/clientpositive/spark/join29.q.out index 88929fe2c9..8477a7ec98 100644 --- ql/src/test/results/clientpositive/spark/join29.q.out +++ ql/src/test/results/clientpositive/spark/join29.q.out @@ -136,7 +136,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq1.key, subq1.cnt, subq2.cnt diff --git ql/src/test/results/clientpositive/spark/join3.q.out ql/src/test/results/clientpositive/spark/join3.q.out index e50f091277..5ee3c52be8 100644 --- ql/src/test/results/clientpositive/spark/join3.q.out +++ ql/src/test/results/clientpositive/spark/join3.q.out @@ -114,7 +114,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) INSERT OVERWRITE TABLE dest1 SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/join30.q.out ql/src/test/results/clientpositive/spark/join30.q.out index 23650ffe1b..5bfa57b393 100644 --- ql/src/test/results/clientpositive/spark/join30.q.out +++ ql/src/test/results/clientpositive/spark/join30.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, count(1) FROM src1 x JOIN src y ON (x.key = y.key) group by x.key diff --git ql/src/test/results/clientpositive/spark/join31.q.out ql/src/test/results/clientpositive/spark/join31.q.out index 4edb4ef976..7eaaff904a 100644 --- ql/src/test/results/clientpositive/spark/join31.q.out +++ ql/src/test/results/clientpositive/spark/join31.q.out @@ -143,7 +143,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT subq1.key, count(1) as cnt diff --git ql/src/test/results/clientpositive/spark/join32.q.out ql/src/test/results/clientpositive/spark/join32.q.out index 3521f2b0e9..80e8f35f67 100644 --- ql/src/test/results/clientpositive/spark/join32.q.out +++ ql/src/test/results/clientpositive/spark/join32.q.out @@ -335,7 +335,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join32_lessSize.q.out ql/src/test/results/clientpositive/spark/join32_lessSize.q.out index 2a6b9b64ff..d3e648f9a6 100644 --- ql/src/test/results/clientpositive/spark/join32_lessSize.q.out +++ ql/src/test/results/clientpositive/spark/join32_lessSize.q.out @@ -343,7 +343,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -880,7 +881,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -1330,7 +1332,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -1778,7 +1781,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 @@ -2035,7 +2039,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, x.value, res.value @@ -2279,7 +2284,8 @@ STAGE PLANS: name: default.dest_j2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j2 SELECT res.key, y.value, res.value diff --git ql/src/test/results/clientpositive/spark/join33.q.out ql/src/test/results/clientpositive/spark/join33.q.out index 3521f2b0e9..80e8f35f67 100644 --- ql/src/test/results/clientpositive/spark/join33.q.out +++ ql/src/test/results/clientpositive/spark/join33.q.out @@ -335,7 +335,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join34.q.out ql/src/test/results/clientpositive/spark/join34.q.out index 503235f607..2c9fb99b9e 100644 --- ql/src/test/results/clientpositive/spark/join34.q.out +++ ql/src/test/results/clientpositive/spark/join34.q.out @@ -337,7 +337,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join35.q.out ql/src/test/results/clientpositive/spark/join35.q.out index 54f68af58a..86ccc5adc1 100644 --- ql/src/test/results/clientpositive/spark/join35.q.out +++ ql/src/test/results/clientpositive/spark/join35.q.out @@ -379,7 +379,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/join36.q.out ql/src/test/results/clientpositive/spark/join36.q.out index b1717e02e3..b469fc306e 100644 --- ql/src/test/results/clientpositive/spark/join36.q.out +++ ql/src/test/results/clientpositive/spark/join36.q.out @@ -138,7 +138,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(x) */ x.key, x.cnt, y.cnt diff --git ql/src/test/results/clientpositive/spark/join37.q.out ql/src/test/results/clientpositive/spark/join37.q.out index 327e93ecd2..8dc4078732 100644 --- ql/src/test/results/clientpositive/spark/join37.q.out +++ ql/src/test/results/clientpositive/spark/join37.q.out @@ -98,7 +98,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(X) */ x.key, x.value, y.value diff --git ql/src/test/results/clientpositive/spark/join39.q.out ql/src/test/results/clientpositive/spark/join39.q.out index 8f0ba62f2b..c094dd9d35 100644 --- ql/src/test/results/clientpositive/spark/join39.q.out +++ ql/src/test/results/clientpositive/spark/join39.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value diff --git ql/src/test/results/clientpositive/spark/join4.q.out ql/src/test/results/clientpositive/spark/join4.q.out index 7e1d4731d7..44ec2e052f 100644 --- ql/src/test/results/clientpositive/spark/join4.q.out +++ ql/src/test/results/clientpositive/spark/join4.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join5.q.out ql/src/test/results/clientpositive/spark/join5.q.out index c354a0ebee..37a96d7b29 100644 --- ql/src/test/results/clientpositive/spark/join5.q.out +++ ql/src/test/results/clientpositive/spark/join5.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join6.q.out ql/src/test/results/clientpositive/spark/join6.q.out index c488247216..563c9407bd 100644 --- ql/src/test/results/clientpositive/spark/join6.q.out +++ ql/src/test/results/clientpositive/spark/join6.q.out @@ -118,7 +118,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join7.q.out ql/src/test/results/clientpositive/spark/join7.q.out index d9b7c0563c..86838cc216 100644 --- ql/src/test/results/clientpositive/spark/join7.q.out +++ ql/src/test/results/clientpositive/spark/join7.q.out @@ -148,7 +148,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join8.q.out ql/src/test/results/clientpositive/spark/join8.q.out index ae308b0d1c..e6b7c1f0d8 100644 --- ql/src/test/results/clientpositive/spark/join8.q.out +++ ql/src/test/results/clientpositive/spark/join8.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM ( FROM diff --git ql/src/test/results/clientpositive/spark/join9.q.out ql/src/test/results/clientpositive/spark/join9.q.out index c7a191a4c0..76534ad20b 100644 --- ql/src/test/results/clientpositive/spark/join9.q.out +++ ql/src/test/results/clientpositive/spark/join9.q.out @@ -252,7 +252,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) diff --git ql/src/test/results/clientpositive/spark/join_map_ppr.q.out ql/src/test/results/clientpositive/spark/join_map_ppr.q.out index 86f3d9a8c4..d2c674e7c1 100644 --- ql/src/test/results/clientpositive/spark/join_map_ppr.q.out +++ ql/src/test/results/clientpositive/spark/join_map_ppr.q.out @@ -319,7 +319,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 @@ -811,7 +812,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest_j1 diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out index de3af40db1..86363070d9 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out @@ -153,7 +153,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out index 6427f70496..10a3892f76 100644 --- ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/spark/list_bucket_dml_2.q.out @@ -204,7 +204,8 @@ STAGE PLANS: name: default.list_bucketing_static_part Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11') diff --git ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out index 99ebfbd390..9c62fb344b 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out @@ -113,7 +113,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -129,7 +130,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out index 24c76c2816..ad4668e149 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out @@ -83,7 +83,8 @@ STAGE PLANS: name: default.nzhang_part10 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part10 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' diff --git ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out index 4318004451..5c7e63cb31 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part13.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.nzhang_part13 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part13 partition (ds="2010-03-03", hr) select * from ( diff --git ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out index 7a562e42b5..08d63506c0 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part14.q.out @@ -135,7 +135,8 @@ STAGE PLANS: name: default.nzhang_part14 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( diff --git ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out index 9bdd77be56..6a7bb4d238 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out @@ -88,7 +88,8 @@ STAGE PLANS: name: default.nzhang_part_bucket Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part_bucket partition (ds='2010-03-23', hr) select key, value, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out index 0d4b21a1be..ca6729aba8 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.nzhang_part3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part3 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out index 937b9d08b3..368c08a501 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out @@ -91,7 +91,8 @@ STAGE PLANS: name: default.nzhang_part4 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part4 partition (ds, hr) select key, value, ds, hr from srcpart where ds is not null and hr is not null PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out index 16da589595..4fc0d8c31a 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.nzhang_part5 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table nzhang_part5 partition (value) select key, value from src PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out index 8322177726..fea2d518ac 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out @@ -366,7 +366,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -398,7 +399,8 @@ STAGE PLANS: name: default.nzhang_part8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from srcpart diff --git ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out index eb242af874..4bb08a05d9 100644 --- ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out +++ ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out @@ -83,7 +83,8 @@ STAGE PLANS: name: default.nzhang_part9 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from srcpart insert overwrite table nzhang_part9 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git ql/src/test/results/clientpositive/spark/mapreduce1.q.out ql/src/test/results/clientpositive/spark/mapreduce1.q.out index d75b482871..6883343f8f 100644 --- ql/src/test/results/clientpositive/spark/mapreduce1.q.out +++ ql/src/test/results/clientpositive/spark/mapreduce1.q.out @@ -82,7 +82,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/spark/mapreduce2.q.out ql/src/test/results/clientpositive/spark/mapreduce2.q.out index adfb503aa7..462a1861b5 100644 --- ql/src/test/results/clientpositive/spark/mapreduce2.q.out +++ ql/src/test/results/clientpositive/spark/mapreduce2.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 diff --git ql/src/test/results/clientpositive/spark/merge1.q.out ql/src/test/results/clientpositive/spark/merge1.q.out index 8e671e952c..eee05db666 100644 --- ql/src/test/results/clientpositive/spark/merge1.q.out +++ ql/src/test/results/clientpositive/spark/merge1.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -563,7 +564,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -673,7 +675,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/merge2.q.out ql/src/test/results/clientpositive/spark/merge2.q.out index 24116cb468..c210fe4970 100644 --- ql/src/test/results/clientpositive/spark/merge2.q.out +++ ql/src/test/results/clientpositive/spark/merge2.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -563,7 +564,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -673,7 +675,8 @@ STAGE PLANS: name: default.test1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out index 2891c7df23..75836509bf 100644 --- ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out +++ ql/src/test/results/clientpositive/spark/metadata_only_queries.q.out @@ -419,40 +419,54 @@ STAGE PLANS: PREHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl +PREHOOK: Output: default@stats_tbl #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl +POSTHOOK: Output: default@stats_tbl #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2011 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2011 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2012 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2012 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### PREHOOK: query: explain select count(*), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si) from stats_tbl diff --git ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out index 6376aa79f8..79d9d27203 100644 --- ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out +++ ql/src/test/results/clientpositive/spark/metadata_only_queries_with_filters.q.out @@ -126,21 +126,29 @@ PREHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2014 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2014 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### PREHOOK: query: explain select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010 diff --git ql/src/test/results/clientpositive/spark/multi_insert.q.out ql/src/test/results/clientpositive/spark/multi_insert.q.out index 4ed1e50b9c..25730b80e8 100644 --- ql/src/test/results/clientpositive/spark/multi_insert.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert.q.out @@ -83,7 +83,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -96,7 +97,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -220,7 +222,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -233,7 +236,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -357,7 +361,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -370,7 +375,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -494,7 +500,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -507,7 +514,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -647,7 +655,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -660,7 +669,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -793,7 +803,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -806,7 +817,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -939,7 +951,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -952,7 +965,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1085,7 +1099,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1098,7 +1113,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1242,7 +1258,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1255,7 +1272,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1425,7 +1443,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1438,7 +1457,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1608,7 +1628,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1621,7 +1642,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1791,7 +1813,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1804,7 +1827,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 diff --git ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out index c86731cfa8..3c478da755 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out @@ -111,7 +111,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -124,7 +125,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 @@ -296,7 +298,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -309,7 +312,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out index 3d67b1d261..abe331e4e7 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_gby2.q.out @@ -101,7 +101,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -114,7 +115,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as value from src order by key) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out index 813704f564..96d404cb65 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -130,7 +131,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -227,7 +229,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -240,7 +243,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key, cast(key as double) as keyD, value from src order by key) a INSERT OVERWRITE TABLE e1 @@ -1677,7 +1681,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1690,7 +1695,8 @@ STAGE PLANS: name: default.e3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: explain FROM (select key, cast(key as double) as keyD, value from src order by key) a @@ -1834,7 +1840,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1847,7 +1854,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1860,5 +1868,6 @@ STAGE PLANS: name: default.e3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out index 7b804daba3..903bf1b087 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -172,7 +173,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C @@ -421,7 +423,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -434,7 +437,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -659,7 +663,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -672,7 +677,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -685,7 +691,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key @@ -969,7 +976,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -982,7 +990,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -995,7 +1004,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C @@ -1340,7 +1350,8 @@ STAGE PLANS: name: default.src_lv1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1353,7 +1364,8 @@ STAGE PLANS: name: default.src_lv2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1366,7 +1378,8 @@ STAGE PLANS: name: default.src_lv3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1379,7 +1392,8 @@ STAGE PLANS: name: default.src_lv4 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src_10 insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key diff --git ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out index 2b28d5313e..56d7d807be 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_mixed.q.out @@ -176,7 +176,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -189,7 +190,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -202,7 +204,8 @@ STAGE PLANS: name: default.src_multi3 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select key, count(1) group by key order by key diff --git ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out index 85d57b14a4..db807833dc 100644 --- ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out +++ ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -100,7 +101,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -228,7 +230,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -241,7 +244,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -369,7 +373,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -382,7 +387,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -510,7 +516,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -523,7 +530,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 @@ -667,7 +675,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -680,7 +689,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -817,7 +827,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -830,7 +841,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -967,7 +979,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -980,7 +993,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1117,7 +1131,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1130,7 +1145,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from src insert overwrite table src_multi1 select * where key < 10 group by key, value @@ -1278,7 +1294,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1291,7 +1308,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1465,7 +1483,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1478,7 +1497,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1652,7 +1672,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1665,7 +1686,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -1839,7 +1861,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1852,7 +1875,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from src union all select * from src) s insert overwrite table src_multi1 select * where key < 10 @@ -2901,7 +2925,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -2914,7 +2939,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3121,7 +3147,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3134,7 +3161,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3341,7 +3369,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3354,7 +3383,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -3561,7 +3591,8 @@ STAGE PLANS: name: default.src_multi1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3574,7 +3605,8 @@ STAGE PLANS: name: default.src_multi2 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator diff --git ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out index a146a8e83b..c34d9511db 100644 --- ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out +++ ql/src/test/results/clientpositive/spark/multigroupby_singlemr.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -170,7 +171,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -291,7 +293,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -304,7 +307,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -425,7 +429,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -438,7 +443,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -531,7 +537,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -544,7 +551,8 @@ STAGE PLANS: name: default.dest4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: EXPLAIN FROM TBL @@ -711,7 +719,8 @@ STAGE PLANS: name: default.dest3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -724,7 +733,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -737,5 +747,6 @@ STAGE PLANS: name: default.dest1 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/orc_merge1.q.out ql/src/test/results/clientpositive/spark/orc_merge1.q.out index 1407616c01..e5dd9d16f7 100644 --- ql/src/test/results/clientpositive/spark/orc_merge1.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge1.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -179,7 +180,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -302,7 +304,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge2.q.out ql/src/test/results/clientpositive/spark/orc_merge2.q.out index b7f1a65e22..e05ba8c733 100644 --- ql/src/test/results/clientpositive/spark/orc_merge2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge2.q.out @@ -81,7 +81,8 @@ STAGE PLANS: name: default.orcfile_merge2a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge3.q.out ql/src/test/results/clientpositive/spark/orc_merge3.q.out index 81a6013d1e..2887a4bb8c 100644 --- ql/src/test/results/clientpositive/spark/orc_merge3.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge3.q.out @@ -107,7 +107,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge4.q.out ql/src/test/results/clientpositive/spark/orc_merge4.q.out index 8d433b031a..fdc601204e 100644 --- ql/src/test/results/clientpositive/spark/orc_merge4.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge4.q.out @@ -125,7 +125,8 @@ STAGE PLANS: name: default.orcfile_merge3b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge5.q.out ql/src/test/results/clientpositive/spark/orc_merge5.q.out index a7adcb43e5..c4006ad535 100644 --- ql/src/test/results/clientpositive/spark/orc_merge5.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge5.q.out @@ -69,7 +69,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -166,7 +167,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -287,7 +289,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5b concatenate PREHOOK: type: ALTER_TABLE_MERGE diff --git ql/src/test/results/clientpositive/spark/orc_merge6.q.out ql/src/test/results/clientpositive/spark/orc_merge6.q.out index 1028618560..66d6ddef2e 100644 --- ql/src/test/results/clientpositive/spark/orc_merge6.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge6.q.out @@ -72,7 +72,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (year="2000",hour=24) select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY @@ -214,7 +215,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -422,7 +424,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(year="2000",hour=24) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/spark/orc_merge7.q.out ql/src/test/results/clientpositive/spark/orc_merge7.q.out index 780f5c8b6e..6fc8211f04 100644 --- ql/src/test/results/clientpositive/spark/orc_merge7.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge7.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 PREHOOK: type: QUERY @@ -243,7 +244,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -528,7 +530,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out index 1407616c01..e5dd9d16f7 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.orcfile_merge1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -179,7 +180,8 @@ STAGE PLANS: name: default.orcfile_merge1b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark @@ -302,7 +304,8 @@ STAGE PLANS: name: default.orcfile_merge1c Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Spark diff --git ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out index 01f8bd47f4..e142d617fe 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_incompat1.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.orc_merge5b Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5b select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out index 0f1db167c0..db77ce4955 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table orc_merge5a partition (st) select userid,string1,subtype,decimal1,ts,subtype from orc_merge5 order by userid PREHOOK: type: QUERY @@ -280,7 +281,8 @@ STAGE PLANS: name: default.orc_merge5a Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: alter table orc_merge5a partition(st=80.0) concatenate PREHOOK: type: ALTER_PARTITION_MERGE diff --git ql/src/test/results/clientpositive/spark/parallel.q.out ql/src/test/results/clientpositive/spark/parallel.q.out index e31fcf0b5a..ecb80622da 100644 --- ql/src/test/results/clientpositive/spark/parallel.q.out +++ ql/src/test/results/clientpositive/spark/parallel.q.out @@ -112,7 +112,8 @@ STAGE PLANS: name: default.src_a Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -125,7 +126,8 @@ STAGE PLANS: name: default.src_b Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select key, value from src group by key, value) s insert overwrite table src_a select s.key, s.value group by s.key, s.value diff --git ql/src/test/results/clientpositive/spark/parallel_join1.q.out ql/src/test/results/clientpositive/spark/parallel_join1.q.out index 7fdd48d2ca..db2962faa7 100644 --- ql/src/test/results/clientpositive/spark/parallel_join1.q.out +++ ql/src/test/results/clientpositive/spark/parallel_join1.q.out @@ -95,7 +95,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/parallel_orderby.q.out ql/src/test/results/clientpositive/spark/parallel_orderby.q.out index cb6c2b0b19..caa6835312 100644 --- ql/src/test/results/clientpositive/spark/parallel_orderby.q.out +++ ql/src/test/results/clientpositive/spark/parallel_orderby.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.total_ordered Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table total_ordered as select * from src5 order by key, value PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/pcr.q.out ql/src/test/results/clientpositive/spark/pcr.q.out index f37323310e..42ecd51322 100644 --- ql/src/test/results/clientpositive/spark/pcr.q.out +++ ql/src/test/results/clientpositive/spark/pcr.q.out @@ -3677,7 +3677,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -3709,7 +3710,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from pcr_t1 @@ -3922,7 +3924,8 @@ STAGE PLANS: name: default.pcr_t2 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-1 @@ -3954,7 +3957,8 @@ STAGE PLANS: name: default.pcr_t3 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: from pcr_t1 diff --git ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out index 38b1f6dadd..e30cf7418d 100644 --- ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out +++ ql/src/test/results/clientpositive/spark/ppd_multi_insert.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -182,7 +183,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -198,7 +200,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator @@ -1442,7 +1445,8 @@ STAGE PLANS: name: default.mi1 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1455,7 +1459,8 @@ STAGE PLANS: name: default.mi2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Move Operator @@ -1471,7 +1476,8 @@ STAGE PLANS: name: default.mi3 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Move Operator diff --git ql/src/test/results/clientpositive/spark/ptf.q.out ql/src/test/results/clientpositive/spark/ptf.q.out index 82fc9f8198..ea599834cd 100644 --- ql/src/test/results/clientpositive/spark/ptf.q.out +++ ql/src/test/results/clientpositive/spark/ptf.q.out @@ -3097,7 +3097,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3110,7 +3111,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part partition by p_mfgr diff --git ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out index 804ff02948..bcf09d87b5 100644 --- ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out +++ ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out @@ -167,7 +167,8 @@ STAGE PLANS: name: default.bucket5_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table bucket5_1 @@ -375,6 +376,7 @@ STAGE PLANS: name: default.complex_tbl_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/sample1.q.out ql/src/test/results/clientpositive/spark/sample1.q.out index ee9eb14685..26d6148ccb 100644 --- ql/src/test/results/clientpositive/spark/sample1.q.out +++ ql/src/test/results/clientpositive/spark/sample1.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample2.q.out ql/src/test/results/clientpositive/spark/sample2.q.out index 85266d1bd8..dff572ea09 100644 --- ql/src/test/results/clientpositive/spark/sample2.q.out +++ ql/src/test/results/clientpositive/spark/sample2.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample4.q.out ql/src/test/results/clientpositive/spark/sample4.q.out index 69e7ee94f8..8c4640f01d 100644 --- ql/src/test/results/clientpositive/spark/sample4.q.out +++ ql/src/test/results/clientpositive/spark/sample4.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample5.q.out ql/src/test/results/clientpositive/spark/sample5.q.out index 558b2dbb68..4af424cc4d 100644 --- ql/src/test/results/clientpositive/spark/sample5.q.out +++ ql/src/test/results/clientpositive/spark/sample5.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample6.q.out ql/src/test/results/clientpositive/spark/sample6.q.out index 4b358291e4..1b49bd32df 100644 --- ql/src/test/results/clientpositive/spark/sample6.q.out +++ ql/src/test/results/clientpositive/spark/sample6.q.out @@ -154,7 +154,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/sample7.q.out ql/src/test/results/clientpositive/spark/sample7.q.out index eae33cad36..98c2d92b3a 100644 --- ql/src/test/results/clientpositive/spark/sample7.q.out +++ ql/src/test/results/clientpositive/spark/sample7.q.out @@ -155,7 +155,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT s.* diff --git ql/src/test/results/clientpositive/spark/skewjoin.q.out ql/src/test/results/clientpositive/spark/skewjoin.q.out index fb6f08d5dc..6ca00d7840 100644 --- ql/src/test/results/clientpositive/spark/skewjoin.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin.q.out @@ -209,7 +209,8 @@ STAGE PLANS: name: default.dest_j1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT OVERWRITE TABLE dest_j1 SELECT src1.key, src2.value diff --git ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out index 3f10ee5685..a0adab1e68 100644 --- ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out +++ ql/src/test/results/clientpositive/spark/skewjoin_noskew.q.out @@ -163,7 +163,8 @@ STAGE PLANS: name: default.noskew Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table noskew as select a.* from src a join src b on a.key=b.key order by a.key limit 30 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out index 9424361c56..054905defd 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out @@ -213,7 +213,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds = '1' diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out index d0bb917eea..6f1ab067cb 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_12.q.out @@ -228,7 +228,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT /*+ MAPJOIN(b) */ a.key, b.value FROM test_table1 a JOIN test_table2 b ON a.key = b.key AND a.ds = '1' AND b.ds >= '1' @@ -444,7 +445,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '2') diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out index 6ed3c21a99..90d0e0ecfd 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_18.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' @@ -267,7 +268,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' and a.key = 238 @@ -385,7 +387,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '2') SELECT a.key, a.value FROM test_table2 a WHERE a.ds = '2' diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out index fb2501597d..71267d6ac4 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value FROM test_table1 a WHERE a.ds = '1' diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out index 6be912398e..314c07c804 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 PARTITION (ds = '1') SELECT a.key, a.value, a.value FROM test_table1 a WHERE a.ds = '1' @@ -214,7 +215,8 @@ STAGE PLANS: name: default.test_table3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.value, a.key, a.value FROM test_table1 a WHERE a.ds = '1' @@ -1382,5 +1384,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out index 1a9118d378..0a63bfb42f 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out @@ -79,7 +79,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -162,7 +163,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -244,7 +246,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -327,7 +330,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -410,7 +414,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: drop table test_table2 PREHOOK: type: DROPTABLE @@ -492,5 +497,6 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out index d8bdef2d25..e4681babac 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_22.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 @@ -218,7 +219,8 @@ STAGE PLANS: name: default.test_table2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_table2 SELECT * FROM test_table1 diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out index 7cf3cf7e64..745e221188 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out @@ -140,7 +140,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -1312,7 +1313,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key @@ -2500,7 +2502,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 @@ -2604,7 +2607,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(b)*/ * from smb_bucket4_1 a join smb_bucket4_2 b on a.key = b.key where a.key>1000 diff --git ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out index 622b950cef..e2f68a02bc 100644 --- ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out +++ ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out @@ -661,7 +661,8 @@ STAGE PLANS: name: default.smb_join_results Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table smb_join_results select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out index 95efa39442..676b28b9fb 100644 --- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out @@ -106,7 +106,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table srcpart_date as select ds as ds, ds as `date` from srcpart group by ds PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out index 76c74d9ab7..0fca7d5a21 100644 --- ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out +++ ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out @@ -45,7 +45,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -78,7 +78,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (GROUP) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.src_orc_merge_test_part"} @@ -3193,7 +3193,7 @@ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-3 Create Table Operator: name:default.nzhang_CTAS1 @@ -3243,7 +3243,7 @@ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT) Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-3 Create Table Operator: name:default.nzhang_ctas3 @@ -4805,7 +4805,7 @@ Reducer 6 <- Map 1 (PARTITION-LEVEL SORT) Reducer 7 <- Map 1 (PARTITION-LEVEL SORT) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.part_4"} @@ -4858,7 +4858,7 @@ Stage-3 Output:["_col1","_col2","_col5","_col7"] <- Please refer to the previous Map 1 [PARTITION-LEVEL SORT] Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.part_5"} @@ -5223,7 +5223,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT) Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0(CONDITIONAL) Move Operator table:{"name:":"default.dest_j1"} diff --git ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out index 371e756eaf..a5068f8d03 100644 --- ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out +++ ql/src/test/results/clientpositive/spark/spark_multi_insert_parallel_orderby.q.out @@ -169,7 +169,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -182,7 +183,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key,value from src order by key limit 10) a INSERT OVERWRITE TABLE e1 @@ -306,7 +308,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -319,7 +322,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key,value from src order by key) a INSERT OVERWRITE TABLE e1 @@ -1442,7 +1446,8 @@ STAGE PLANS: name: default.e1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1455,7 +1460,8 @@ STAGE PLANS: name: default.e2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select key,value from src order by key) a INSERT OVERWRITE TABLE e1 diff --git ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out index 0a3ba5f830..d9ea36f996 100644 --- ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out +++ ql/src/test/results/clientpositive/spark/spark_use_ts_stats_for_mapjoin.q.out @@ -230,7 +230,8 @@ STAGE PLANS: name: default.dest Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) INSERT OVERWRITE TABLE dest SELECT src1.key, src3.value diff --git ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out index 514c77cf1f..80844fc36e 100644 --- ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out +++ ql/src/test/results/clientpositive/spark/spark_vectorized_dynamic_partition_pruning.q.out @@ -108,7 +108,8 @@ STAGE PLANS: name: default.srcpart_date Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table srcpart_date stored as orc as select ds as ds, ds as `date` from srcpart group by ds PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/stats0.q.out ql/src/test/results/clientpositive/spark/stats0.q.out index 66dfa631fd..0fae83b7d6 100644 --- ql/src/test/results/clientpositive/spark/stats0.q.out +++ ql/src/test/results/clientpositive/spark/stats0.q.out @@ -147,7 +147,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table stats_non_partitioned @@ -737,7 +738,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src @@ -1463,7 +1465,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table stats_non_partitioned @@ -2053,7 +2056,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src diff --git ql/src/test/results/clientpositive/spark/stats1.q.out ql/src/test/results/clientpositive/spark/stats1.q.out index cdabdce394..1997996856 100644 --- ql/src/test/results/clientpositive/spark/stats1.q.out +++ ql/src/test/results/clientpositive/spark/stats1.q.out @@ -96,7 +96,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE tmptable SELECT unionsrc.key, unionsrc.value @@ -212,8 +213,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 3 - numRows 26 - rawDataSize 199 totalSize 1583 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/stats10.q.out ql/src/test/results/clientpositive/spark/stats10.q.out index 865a3b8320..d79fc101d2 100644 --- ql/src/test/results/clientpositive/spark/stats10.q.out +++ ql/src/test/results/clientpositive/spark/stats10.q.out @@ -68,7 +68,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') select * from src @@ -383,7 +384,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table bucket3_1 partition (ds) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats12.q.out ql/src/test/results/clientpositive/spark/stats12.q.out index 9c96819ced..484a0458ef 100644 --- ql/src/test/results/clientpositive/spark/stats12.q.out +++ ql/src/test/results/clientpositive/spark/stats12.q.out @@ -153,8 +153,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=12 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats13.q.out ql/src/test/results/clientpositive/spark/stats13.q.out index 9e4302bc81..c410898ceb 100644 --- ql/src/test/results/clientpositive/spark/stats13.q.out +++ ql/src/test/results/clientpositive/spark/stats13.q.out @@ -107,8 +107,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=11 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats18.q.out ql/src/test/results/clientpositive/spark/stats18.q.out index dd54c0dbc3..a39a18d60c 100644 --- ql/src/test/results/clientpositive/spark/stats18.q.out +++ ql/src/test/results/clientpositive/spark/stats18.q.out @@ -84,8 +84,6 @@ Table: stats_part #### A masked pattern was here #### Partition Parameters: numFiles 2 - numRows 500 - rawDataSize 5312 totalSize 7170 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/stats2.q.out ql/src/test/results/clientpositive/spark/stats2.q.out index 74e867890f..f609702e01 100644 --- ql/src/test/results/clientpositive/spark/stats2.q.out +++ ql/src/test/results/clientpositive/spark/stats2.q.out @@ -138,7 +138,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_t1 partition (ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats3.q.out ql/src/test/results/clientpositive/spark/stats3.q.out index 7d4ed9b60b..71392b8a66 100644 --- ql/src/test/results/clientpositive/spark/stats3.q.out +++ ql/src/test/results/clientpositive/spark/stats3.q.out @@ -54,7 +54,8 @@ STAGE PLANS: name: default.hive_test_src Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src PREHOOK: type: LOAD @@ -81,8 +82,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 11 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/stats5.q.out ql/src/test/results/clientpositive/spark/stats5.q.out index 6684abe853..120420082e 100644 --- ql/src/test/results/clientpositive/spark/stats5.q.out +++ ql/src/test/results/clientpositive/spark/stats5.q.out @@ -30,7 +30,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_src compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats7.q.out ql/src/test/results/clientpositive/spark/stats7.q.out index 9c5a71c414..13ca9687d6 100644 --- ql/src/test/results/clientpositive/spark/stats7.q.out +++ ql/src/test/results/clientpositive/spark/stats7.q.out @@ -53,7 +53,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats8.q.out ql/src/test/results/clientpositive/spark/stats8.q.out index b83adbd0de..8dedb62eaf 100644 --- ql/src/test/results/clientpositive/spark/stats8.q.out +++ ql/src/test/results/clientpositive/spark/stats8.q.out @@ -53,7 +53,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY @@ -164,7 +165,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics PREHOOK: type: QUERY @@ -236,7 +238,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics PREHOOK: type: QUERY @@ -308,7 +311,8 @@ STAGE PLANS: Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics PREHOOK: type: QUERY @@ -380,7 +384,8 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats9.q.out ql/src/test/results/clientpositive/spark/stats9.q.out index c2ca23fab8..08590a5cd3 100644 --- ql/src/test/results/clientpositive/spark/stats9.q.out +++ ql/src/test/results/clientpositive/spark/stats9.q.out @@ -36,7 +36,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11603 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcbucket compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out index 6d3f413956..c8fa18cb22 100644 --- ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out +++ ql/src/test/results/clientpositive/spark/stats_noscan_1.q.out @@ -44,7 +44,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan PREHOOK: type: QUERY @@ -305,7 +306,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/stats_only_null.q.out ql/src/test/results/clientpositive/spark/stats_only_null.q.out index be906f6403..758503f777 100644 --- ql/src/test/results/clientpositive/spark/stats_only_null.q.out +++ ql/src/test/results/clientpositive/spark/stats_only_null.q.out @@ -181,30 +181,40 @@ STAGE PLANS: PREHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null +PREHOOK: Output: default@stats_null #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null +POSTHOOK: Output: default@stats_null #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2010 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2010 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2011 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2011 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition (dt='2010') PREHOOK: type: DESCTABLE @@ -372,12 +382,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=1 PREHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=1 +PREHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=1 POSTHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=1 +POSTHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition(dt = 1) a PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out new file mode 100644 index 0000000000..403bc28cfa --- /dev/null +++ ql/src/test/results/clientpositive/spark/stats_partscan_1_23.q.out @@ -0,0 +1,188 @@ +PREHOOK: query: CREATE table analyze_srcpart_partial_scan (key STRING, value STRING) +partitioned by (ds string, hr string) +stored as rcfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: CREATE table analyze_srcpart_partial_scan (key STRING, value STRING) +partitioned by (ds string, hr string) +stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart_partial_scan +PREHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain +analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +PREHOOK: type: QUERY +POSTHOOK: query: explain +analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-0, Stage-2 + +STAGE PLANS: + Stage: Stage-2 + Partial Scan Statistics + + Stage: Stage-1 + Stats Work + Basic Stats Work: + +PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart_partial_scan +PREHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart_partial_scan +PREHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table analyze_srcpart_partial_scan +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: drop table analyze_srcpart_partial_scan +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: Output: default@analyze_srcpart_partial_scan diff --git ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out index a0adcaee08..5b4aa49900 100644 --- ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.out @@ -257,7 +257,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-0 Move Operator @@ -270,7 +271,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Shuffle Join JOIN[31][tables = [sq_2_notin_nullcheck]] in Work 'Reducer 2' is a cross product PREHOOK: query: from src b @@ -680,7 +682,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -693,7 +696,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Map Join MAPJOIN[47][bigTable=b] in task 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b diff --git ql/src/test/results/clientpositive/spark/temp_table.q.out ql/src/test/results/clientpositive/spark/temp_table.q.out index c2ec3b4e21..342d08c1a2 100644 --- ql/src/test/results/clientpositive/spark/temp_table.q.out +++ ql/src/test/results/clientpositive/spark/temp_table.q.out @@ -52,7 +52,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TEMPORARY TABLE foo AS SELECT * FROM src WHERE key % 2 = 0 PREHOOK: type: CREATETABLE_AS_SELECT @@ -118,7 +119,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: CREATE TEMPORARY TABLE bar AS SELECT * FROM src WHERE key % 2 = 1 PREHOOK: type: CREATETABLE_AS_SELECT diff --git ql/src/test/results/clientpositive/spark/union10.q.out ql/src/test/results/clientpositive/spark/union10.q.out index ea1bebb8b7..c83afda688 100644 --- ql/src/test/results/clientpositive/spark/union10.q.out +++ ql/src/test/results/clientpositive/spark/union10.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union12.q.out ql/src/test/results/clientpositive/spark/union12.q.out index 063995660c..7d4ffb51a9 100644 --- ql/src/test/results/clientpositive/spark/union12.q.out +++ ql/src/test/results/clientpositive/spark/union12.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union17.q.out ql/src/test/results/clientpositive/spark/union17.q.out index a967c0763a..bc72b6cc3f 100644 --- ql/src/test/results/clientpositive/spark/union17.q.out +++ ql/src/test/results/clientpositive/spark/union17.q.out @@ -197,7 +197,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -210,7 +211,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/spark/union18.q.out ql/src/test/results/clientpositive/spark/union18.q.out index 653c54c76e..124dbdc87c 100644 --- ql/src/test/results/clientpositive/spark/union18.q.out +++ ql/src/test/results/clientpositive/spark/union18.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -143,7 +144,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/spark/union19.q.out ql/src/test/results/clientpositive/spark/union19.q.out index fe5902f395..4d2d56a5a3 100644 --- ql/src/test/results/clientpositive/spark/union19.q.out +++ ql/src/test/results/clientpositive/spark/union19.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -172,7 +173,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/spark/union22.q.out ql/src/test/results/clientpositive/spark/union22.q.out index 6acaba8f70..9a8e55308e 100644 --- ql/src/test/results/clientpositive/spark/union22.q.out +++ ql/src/test/results/clientpositive/spark/union22.q.out @@ -407,7 +407,8 @@ STAGE PLANS: name: default.dst_union22 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table dst_union22 partition (ds='2') diff --git ql/src/test/results/clientpositive/spark/union25.q.out ql/src/test/results/clientpositive/spark/union25.q.out index 559b318717..c5a001a441 100644 --- ql/src/test/results/clientpositive/spark/union25.q.out +++ ql/src/test/results/clientpositive/spark/union25.q.out @@ -189,5 +189,6 @@ STAGE PLANS: name: default.tmp_unionall Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/union28.q.out ql/src/test/results/clientpositive/spark/union28.q.out index 7ee06fef70..83c105d569 100644 --- ql/src/test/results/clientpositive/spark/union28.q.out +++ ql/src/test/results/clientpositive/spark/union28.q.out @@ -135,7 +135,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( diff --git ql/src/test/results/clientpositive/spark/union29.q.out ql/src/test/results/clientpositive/spark/union29.q.out index 05c44d1768..3fc229bf7b 100644 --- ql/src/test/results/clientpositive/spark/union29.q.out +++ ql/src/test/results/clientpositive/spark/union29.q.out @@ -117,7 +117,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( diff --git ql/src/test/results/clientpositive/spark/union30.q.out ql/src/test/results/clientpositive/spark/union30.q.out index 9d827eb814..87ae31153a 100644 --- ql/src/test/results/clientpositive/spark/union30.q.out +++ ql/src/test/results/clientpositive/spark/union30.q.out @@ -170,7 +170,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_subq_union select * from ( diff --git ql/src/test/results/clientpositive/spark/union31.q.out ql/src/test/results/clientpositive/spark/union31.q.out index 10f8bdb242..0b02386f99 100644 --- ql/src/test/results/clientpositive/spark/union31.q.out +++ ql/src/test/results/clientpositive/spark/union31.q.out @@ -235,7 +235,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -248,7 +249,8 @@ STAGE PLANS: name: default.t4 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from t1 @@ -488,7 +490,8 @@ STAGE PLANS: name: default.t5 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -501,7 +504,8 @@ STAGE PLANS: name: default.t6 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( @@ -772,7 +776,8 @@ STAGE PLANS: name: default.t7 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -785,7 +790,8 @@ STAGE PLANS: name: default.t8 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( diff --git ql/src/test/results/clientpositive/spark/union33.q.out ql/src/test/results/clientpositive/spark/union33.q.out index def5f69305..8e3e2a20f0 100644 --- ql/src/test/results/clientpositive/spark/union33.q.out +++ ql/src/test/results/clientpositive/spark/union33.q.out @@ -124,7 +124,8 @@ STAGE PLANS: name: default.test_src Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( @@ -277,7 +278,8 @@ STAGE PLANS: name: default.test_src Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_src SELECT key, value FROM ( diff --git ql/src/test/results/clientpositive/spark/union4.q.out ql/src/test/results/clientpositive/spark/union4.q.out index cb8c6a2cb8..f78e82da1a 100644 --- ql/src/test/results/clientpositive/spark/union4.q.out +++ ql/src/test/results/clientpositive/spark/union4.q.out @@ -121,7 +121,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union6.q.out ql/src/test/results/clientpositive/spark/union6.q.out index 6f61839c6e..91c6b2b90e 100644 --- ql/src/test/results/clientpositive/spark/union6.q.out +++ ql/src/test/results/clientpositive/spark/union6.q.out @@ -94,7 +94,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table tmptable select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 diff --git ql/src/test/results/clientpositive/spark/union_lateralview.q.out ql/src/test/results/clientpositive/spark/union_lateralview.q.out index fe9afb8f88..8835a9811f 100644 --- ql/src/test/results/clientpositive/spark/union_lateralview.q.out +++ ql/src/test/results/clientpositive/spark/union_lateralview.q.out @@ -196,7 +196,8 @@ STAGE PLANS: name: default.test_union_lateral_view Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view SELECT b.key, d.arr_ele, d.value diff --git ql/src/test/results/clientpositive/spark/union_top_level.q.out ql/src/test/results/clientpositive/spark/union_top_level.q.out index 6adf6c43e5..86e92d2463 100644 --- ql/src/test/results/clientpositive/spark/union_top_level.q.out +++ ql/src/test/results/clientpositive/spark/union_top_level.q.out @@ -497,7 +497,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table union_top as select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a @@ -709,7 +710,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a @@ -913,7 +915,8 @@ STAGE PLANS: name: default.union_top Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table union_top select * from (select key, 0 as value from src where key % 3 == 0 limit 3)a diff --git ql/src/test/results/clientpositive/spark/vector_char_4.q.out ql/src/test/results/clientpositive/spark/vector_char_4.q.out index 943a4b1423..80e5357837 100644 --- ql/src/test/results/clientpositive/spark/vector_char_4.q.out +++ ql/src/test/results/clientpositive/spark/vector_char_4.q.out @@ -190,5 +190,6 @@ STAGE PLANS: name: default.char_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out index a93c8332bd..101c751741 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out @@ -186,10 +186,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out index bb922e7f2a..c11406d47a 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out index dbbfd34d37..f8d1ec2425 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join3.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out index ffce9e6671..a55250b9f4 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join4.q.out @@ -201,10 +201,12 @@ POSTHOOK: Output: default@small_alltypesorc_b PREHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_b +PREHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b +POSTHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_b PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out index 4f25253ef7..680ee42bb6 100644 --- ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out +++ ql/src/test/results/clientpositive/spark/vector_outer_join5.q.out @@ -27,10 +27,12 @@ POSTHOOK: Output: default@sorted_mod_4 PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@sorted_mod_4 +PREHOOK: Output: default@sorted_mod_4 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@sorted_mod_4 +POSTHOOK: Output: default@sorted_mod_4 #### A masked pattern was here #### PREHOOK: query: create table small_table stored as orc as select ctinyint, cbigint from alltypesorc limit 100 @@ -57,10 +59,12 @@ POSTHOOK: Output: default@small_table PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_table +PREHOOK: Output: default@small_table #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table +POSTHOOK: Output: default@small_table #### A masked pattern was here #### PREHOOK: query: explain vectorization detail formatted select count(*) from (select s.*, st.* @@ -267,10 +271,12 @@ POSTHOOK: Output: default@mod_8_mod_4 PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Output: default@mod_8_mod_4 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Output: default@mod_8_mod_4 #### A masked pattern was here #### PREHOOK: query: create table small_table2 stored as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100 @@ -297,10 +303,12 @@ POSTHOOK: Output: default@small_table2 PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_table2 +PREHOOK: Output: default@small_table2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table2 +POSTHOOK: Output: default@small_table2 #### A masked pattern was here #### PREHOOK: query: explain vectorization detail formatted select count(*) from (select s.*, st.* diff --git ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out index 1c8e479512..1ab7aee6f7 100644 --- ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/spark/vector_varchar_4.q.out @@ -190,5 +190,6 @@ STAGE PLANS: name: default.varchar_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index b231d2cc0a..540a2c8aa1 100644 --- ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -3982,7 +3982,8 @@ STAGE PLANS: name: default.part_4 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -3995,7 +3996,8 @@ STAGE PLANS: name: default.part_5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from noop(on part_orc partition by p_mfgr diff --git ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out index fc5bab0722..8f463e8eec 100644 --- ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out +++ ql/src/test/results/clientpositive/special_character_in_tabnames_2.q.out @@ -29,10 +29,12 @@ POSTHOOK: Output: default@s/c PREHOOK: query: ANALYZE TABLE `s/c` COMPUTE STATISTICS FOR COLUMNS key,value PREHOOK: type: QUERY PREHOOK: Input: default@s/c +PREHOOK: Output: default@s/c #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE `s/c` COMPUTE STATISTICS FOR COLUMNS key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@s/c +POSTHOOK: Output: default@s/c #### A masked pattern was here #### PREHOOK: query: SELECT key, value FROM `s/c` WHERE key > 80 AND key < 100 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats0.q.out ql/src/test/results/clientpositive/stats0.q.out index c7e8da9061..e993eeb663 100644 --- ql/src/test/results/clientpositive/stats0.q.out +++ ql/src/test/results/clientpositive/stats0.q.out @@ -144,7 +144,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### PREHOOK: query: insert overwrite table stats_non_partitioned @@ -731,7 +732,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table stats_partitioned partition (ds='1') select * from src @@ -1468,7 +1470,8 @@ STAGE PLANS: name: default.stats_non_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 @@ -2249,7 +2252,8 @@ STAGE PLANS: name: default.stats_partitioned Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/stats1.q.out ql/src/test/results/clientpositive/stats1.q.out index 075e16bb16..d717380d37 100644 --- ql/src/test/results/clientpositive/stats1.q.out +++ ql/src/test/results/clientpositive/stats1.q.out @@ -104,7 +104,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE tmptable SELECT unionsrc.key, unionsrc.value @@ -220,8 +221,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 3 - numRows 26 - rawDataSize 199 totalSize 1583 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/stats10.q.out ql/src/test/results/clientpositive/stats10.q.out index fa6142fe7d..acd482bccc 100644 --- ql/src/test/results/clientpositive/stats10.q.out +++ ql/src/test/results/clientpositive/stats10.q.out @@ -62,7 +62,8 @@ STAGE PLANS: name: default.bucket3_1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert overwrite table bucket3_1 partition (ds='1') select * from src @@ -374,7 +375,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table bucket3_1 partition (ds) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats12.q.out ql/src/test/results/clientpositive/stats12.q.out index 3b91cc9c54..d6380a477d 100644 --- ql/src/test/results/clientpositive/stats12.q.out +++ ql/src/test/results/clientpositive/stats12.q.out @@ -150,8 +150,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=12 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats13.q.out ql/src/test/results/clientpositive/stats13.q.out index 903e256e1c..a2d9d3e40a 100644 --- ql/src/test/results/clientpositive/stats13.q.out +++ ql/src/test/results/clientpositive/stats13.q.out @@ -104,8 +104,9 @@ STAGE PLANS: /analyze_srcpart/ds=2008-04-08/hr=11 [analyze_srcpart] Stage: Stage-1 - Stats-Aggr Operator - Stats Aggregation Key Prefix: default.analyze_srcpart/ + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.analyze_srcpart/ PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats18.q.out ql/src/test/results/clientpositive/stats18.q.out index dd54c0dbc3..a39a18d60c 100644 --- ql/src/test/results/clientpositive/stats18.q.out +++ ql/src/test/results/clientpositive/stats18.q.out @@ -84,8 +84,6 @@ Table: stats_part #### A masked pattern was here #### Partition Parameters: numFiles 2 - numRows 500 - rawDataSize 5312 totalSize 7170 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/stats2.q.out ql/src/test/results/clientpositive/stats2.q.out index 991b13715a..65691d319c 100644 --- ql/src/test/results/clientpositive/stats2.q.out +++ ql/src/test/results/clientpositive/stats2.q.out @@ -132,7 +132,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_t1 partition (ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats3.q.out ql/src/test/results/clientpositive/stats3.q.out index 7d4ed9b60b..71392b8a66 100644 --- ql/src/test/results/clientpositive/stats3.q.out +++ ql/src/test/results/clientpositive/stats3.q.out @@ -54,7 +54,8 @@ STAGE PLANS: name: default.hive_test_src Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table hive_test_src PREHOOK: type: LOAD @@ -81,8 +82,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 0 - rawDataSize 0 totalSize 11 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/stats4.q.out ql/src/test/results/clientpositive/stats4.q.out index c0c3c12e35..741e47b948 100644 --- ql/src/test/results/clientpositive/stats4.q.out +++ ql/src/test/results/clientpositive/stats4.q.out @@ -120,7 +120,8 @@ STAGE PLANS: name: default.nzhang_part1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -175,7 +176,8 @@ STAGE PLANS: name: default.nzhang_part2 Stage: Stage-9 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-10 Map Reduce diff --git ql/src/test/results/clientpositive/stats5.q.out ql/src/test/results/clientpositive/stats5.q.out index b71c7c1834..8d4f9c4dd2 100644 --- ql/src/test/results/clientpositive/stats5.q.out +++ ql/src/test/results/clientpositive/stats5.q.out @@ -27,7 +27,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_src compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats7.q.out ql/src/test/results/clientpositive/stats7.q.out index 5db261950c..0be8848548 100644 --- ql/src/test/results/clientpositive/stats7.q.out +++ ql/src/test/results/clientpositive/stats7.q.out @@ -50,7 +50,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats8.q.out ql/src/test/results/clientpositive/stats8.q.out index 3a617d4b7d..6969ca8fbf 100644 --- ql/src/test/results/clientpositive/stats8.q.out +++ ql/src/test/results/clientpositive/stats8.q.out @@ -50,7 +50,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics PREHOOK: type: QUERY @@ -158,7 +159,8 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=12) compute statistics PREHOOK: type: QUERY @@ -227,7 +229,8 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=11) compute statistics PREHOOK: type: QUERY @@ -296,7 +299,8 @@ STAGE PLANS: Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-09',hr=12) compute statistics PREHOOK: type: QUERY @@ -365,7 +369,8 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds, hr) compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats9.q.out ql/src/test/results/clientpositive/stats9.q.out index 7556c0521c..d056b29054 100644 --- ql/src/test/results/clientpositive/stats9.q.out +++ ql/src/test/results/clientpositive/stats9.q.out @@ -33,7 +33,8 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 11603 Basic stats: COMPLETE Column stats: COMPLETE Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcbucket compute statistics PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out index b855b3896e..54566546bb 100644 --- ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out +++ ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out @@ -65,7 +65,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/stats_invalidation.q.out ql/src/test/results/clientpositive/stats_invalidation.q.out index 77d39d4b5a..8200582784 100644 --- ql/src/test/results/clientpositive/stats_invalidation.q.out +++ ql/src/test/results/clientpositive/stats_invalidation.q.out @@ -21,10 +21,12 @@ POSTHOOK: Lineage: stats_invalid.value SIMPLE [(src)src.FieldSchema(name:value, PREHOOK: query: analyze table stats_invalid compute statistics for columns key,value PREHOOK: type: QUERY PREHOOK: Input: default@stats_invalid +PREHOOK: Output: default@stats_invalid #### A masked pattern was here #### POSTHOOK: query: analyze table stats_invalid compute statistics for columns key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_invalid +POSTHOOK: Output: default@stats_invalid #### A masked pattern was here #### PREHOOK: query: desc formatted stats_invalid PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/stats_missing_warning.q.out ql/src/test/results/clientpositive/stats_missing_warning.q.out index 0ed70a0960..b90578597b 100644 --- ql/src/test/results/clientpositive/stats_missing_warning.q.out +++ ql/src/test/results/clientpositive/stats_missing_warning.q.out @@ -117,26 +117,32 @@ POSTHOOK: Input: default@missing_stats_t3 PREHOOK: query: ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@missing_stats_t1 +PREHOOK: Output: default@missing_stats_t1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE missing_stats_t1 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@missing_stats_t1 +POSTHOOK: Output: default@missing_stats_t1 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@missing_stats_t2 +PREHOOK: Output: default@missing_stats_t2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE missing_stats_t2 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@missing_stats_t2 +POSTHOOK: Output: default@missing_stats_t2 #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@missing_stats_t3 +PREHOOK: Output: default@missing_stats_t3 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE missing_stats_t3 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@missing_stats_t3 +POSTHOOK: Output: default@missing_stats_t3 #### A masked pattern was here #### PREHOOK: query: SELECT COUNT(*) FROM missing_stats_t1 t1 diff --git ql/src/test/results/clientpositive/stats_noscan_1.q.out ql/src/test/results/clientpositive/stats_noscan_1.q.out index 6d3f413956..c8fa18cb22 100644 --- ql/src/test/results/clientpositive/stats_noscan_1.q.out +++ ql/src/test/results/clientpositive/stats_noscan_1.q.out @@ -44,7 +44,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart PARTITION(ds='2008-04-08',hr=11) compute statistics noscan PREHOOK: type: QUERY @@ -305,7 +306,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: analyze table analyze_srcpart_partial PARTITION(ds='2008-04-08') compute statistics noscan PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_only_null.q.out ql/src/test/results/clientpositive/stats_only_null.q.out index bcbc3f47b5..5bab96000e 100644 --- ql/src/test/results/clientpositive/stats_only_null.q.out +++ ql/src/test/results/clientpositive/stats_only_null.q.out @@ -169,30 +169,40 @@ STAGE PLANS: PREHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null +PREHOOK: Output: default@stats_null #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null +POSTHOOK: Output: default@stats_null #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2010 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2010 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2011 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2011 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition (dt='2010') PREHOOK: type: DESCTABLE @@ -360,12 +370,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=1 PREHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=1 +PREHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=1 POSTHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=1 +POSTHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition(dt = 1) a PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/stats_partial_size.q.out ql/src/test/results/clientpositive/stats_partial_size.q.out index 4a4061db7e..09129f2410 100644 --- ql/src/test/results/clientpositive/stats_partial_size.q.out +++ ql/src/test/results/clientpositive/stats_partial_size.q.out @@ -28,10 +28,12 @@ POSTHOOK: Output: default@sample PREHOOK: query: analyze table sample compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@sample +PREHOOK: Output: default@sample #### A masked pattern was here #### POSTHOOK: query: analyze table sample compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@sample +POSTHOOK: Output: default@sample #### A masked pattern was here #### PREHOOK: query: explain select sample_partitioned.x from sample_partitioned, sample where sample.y = sample_partitioned.y PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/stats_partscan_1_23.q.out ql/src/test/results/clientpositive/stats_partscan_1_23.q.out new file mode 100644 index 0000000000..34f70235cb --- /dev/null +++ ql/src/test/results/clientpositive/stats_partscan_1_23.q.out @@ -0,0 +1,191 @@ +PREHOOK: query: CREATE table analyze_srcpart_partial_scan (key STRING, value STRING) +partitioned by (ds string, hr string) +stored as rcfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: CREATE table analyze_srcpart_partial_scan (key STRING, value STRING) +partitioned by (ds string, hr string) +stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@analyze_srcpart_partial_scan +PREHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: insert overwrite table analyze_srcpart_partial_scan partition (ds, hr) select * from srcpart where ds is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: analyze_srcpart_partial_scan PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain +analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +PREHOOK: type: QUERY +POSTHOOK: query: explain +analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-2 + Partial Scan Statistics + + Stage: Stage-1 + Stats Work + Basic Stats Work: + +PREHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +PREHOOK: type: QUERY +PREHOOK: Input: default@analyze_srcpart_partial_scan +PREHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +PREHOOK: Output: default@analyze_srcpart_partial_scan +PREHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) compute statistics partialscan +POSTHOOK: type: QUERY +POSTHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: Input: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: Output: default@analyze_srcpart_partial_scan@ds=2008-04-08/hr=11 +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-08',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 500 + rawDataSize 4812 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: query: describe formatted analyze_srcpart_partial_scan PARTITION(ds='2008-04-09',hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +# col_name data_type comment + +key string +value string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-09, 11] +Database: default +Table: analyze_srcpart_partial_scan +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + totalSize 5293 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table analyze_srcpart_partial_scan +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@analyze_srcpart_partial_scan +PREHOOK: Output: default@analyze_srcpart_partial_scan +POSTHOOK: query: drop table analyze_srcpart_partial_scan +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@analyze_srcpart_partial_scan +POSTHOOK: Output: default@analyze_srcpart_partial_scan diff --git ql/src/test/results/clientpositive/stats_ppr_all.q.out ql/src/test/results/clientpositive/stats_ppr_all.q.out index 3b9bb30ffa..f19e3f5c2c 100644 --- ql/src/test/results/clientpositive/stats_ppr_all.q.out +++ ql/src/test/results/clientpositive/stats_ppr_all.q.out @@ -46,6 +46,10 @@ PREHOOK: Input: default@ss PREHOOK: Input: default@ss@country=US/year=2015/month=1/day=1 PREHOOK: Input: default@ss@country=US/year=2015/month=1/day=2 PREHOOK: Input: default@ss@country=US/year=2015/month=2/day=1 +PREHOOK: Output: default@ss +PREHOOK: Output: default@ss@country=US/year=2015/month=1/day=1 +PREHOOK: Output: default@ss@country=US/year=2015/month=1/day=2 +PREHOOK: Output: default@ss@country=US/year=2015/month=2/day=1 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE ss PARTITION(country,year,month,day) compute statistics for columns POSTHOOK: type: QUERY @@ -53,6 +57,10 @@ POSTHOOK: Input: default@ss POSTHOOK: Input: default@ss@country=US/year=2015/month=1/day=1 POSTHOOK: Input: default@ss@country=US/year=2015/month=1/day=2 POSTHOOK: Input: default@ss@country=US/year=2015/month=2/day=1 +POSTHOOK: Output: default@ss +POSTHOOK: Output: default@ss@country=US/year=2015/month=1/day=1 +POSTHOOK: Output: default@ss@country=US/year=2015/month=1/day=2 +POSTHOOK: Output: default@ss@country=US/year=2015/month=2/day=1 #### A masked pattern was here #### PREHOOK: query: explain select sum(order_amount) from ss where (country="US" and year=2015 and month=2 and day=1) PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/subquery_multiinsert.q.out ql/src/test/results/clientpositive/subquery_multiinsert.q.out index 28c82b85d2..7abb6b293d 100644 --- ql/src/test/results/clientpositive/subquery_multiinsert.q.out +++ ql/src/test/results/clientpositive/subquery_multiinsert.q.out @@ -233,7 +233,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -293,7 +294,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product PREHOOK: query: from src b @@ -699,7 +701,8 @@ STAGE PLANS: name: default.src_5 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-16 Map Reduce Local Work @@ -763,7 +766,8 @@ STAGE PLANS: name: default.src_4 Stage: Stage-7 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-2 Map Reduce diff --git ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out index 1fb4842f38..cd57ad235b 100644 --- ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out +++ ql/src/test/results/clientpositive/temp_table_display_colstats_tbllvl.q.out @@ -34,6 +34,48 @@ POSTHOOK: type: CREATETABLE #### A masked pattern was here #### POSTHOOK: Output: database:default POSTHOOK: Output: default@UserVisits_web_text_none +PREHOOK: query: desc formatted UserVisits_web_text_none +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@uservisits_web_text_none +POSTHOOK: query: desc formatted UserVisits_web_text_none +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@uservisits_web_text_none +# col_name data_type comment +sourceip string +desturl string +visitdate string +adrevenue float +useragent string +ccode string +lcode string +skeyword string +avgtimeonsite int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adrevenue\":\"true\",\"avgtimeonsite\":\"true\",\"ccode\":\"true\",\"desturl\":\"true\",\"lcode\":\"true\",\"skeyword\":\"true\",\"sourceip\":\"true\",\"useragent\":\"true\",\"visitdate\":\"true\"}} + EXTERNAL TRUE + numFiles 0 + numRows 0 + rawDataSize 0 + totalSize 0 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim | + serialization.format | PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/UserVisits.dat" INTO TABLE UserVisits_web_text_none PREHOOK: type: LOAD #### A masked pattern was here #### @@ -42,6 +84,45 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/UserVisits.dat" INTO T POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@uservisits_web_text_none +PREHOOK: query: desc formatted UserVisits_web_text_none +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@uservisits_web_text_none +POSTHOOK: query: desc formatted UserVisits_web_text_none +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@uservisits_web_text_none +# col_name data_type comment +sourceip string +desturl string +visitdate string +adrevenue float +useragent string +ccode string +lcode string +skeyword string +avgtimeonsite int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + EXTERNAL TRUE + numFiles 1 + totalSize 7060 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim | + serialization.format | PREHOOK: query: desc extended UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -112,7 +193,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -135,7 +217,8 @@ STAGE PLANS: TableScan alias: uservisits_web_text_none Statistics: Num rows: 65 Data size: 7060 Basic stats: COMPLETE Column stats: NONE - GatherStats: false + Statistics Aggregation Key Prefix: default.uservisits_web_text_none/ + GatherStats: true Select Operator expressions: sourceip (type: string), adrevenue (type: float), avgtimeonsite (type: int) outputColumnNames: sourceip, adrevenue, avgtimeonsite @@ -230,7 +313,9 @@ STAGE PLANS: MultiFileSpray: false Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: + Stats Aggregation Key Prefix: default.uservisits_web_text_none/ Column Stats Desc: Columns: sourceIP, avgTimeOnSite, adRevenue Column Types: string, int, float @@ -240,11 +325,55 @@ STAGE PLANS: PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue PREHOOK: type: QUERY PREHOOK: Input: default@uservisits_web_text_none +PREHOOK: Output: default@uservisits_web_text_none #### A masked pattern was here #### POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sourceIP, avgTimeOnSite, adRevenue POSTHOOK: type: QUERY POSTHOOK: Input: default@uservisits_web_text_none +POSTHOOK: Output: default@uservisits_web_text_none +#### A masked pattern was here #### +PREHOOK: query: desc formatted UserVisits_web_text_none +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@uservisits_web_text_none +POSTHOOK: query: desc formatted UserVisits_web_text_none +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@uservisits_web_text_none +# col_name data_type comment +sourceip string +desturl string +visitdate string +adrevenue float +useragent string +ccode string +lcode string +skeyword string +avgtimeonsite int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 #### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} + EXTERNAL TRUE + numFiles 1 + numRows 55 + rawDataSize 7005 + totalSize 7060 + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + field.delim | + serialization.format | PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -263,6 +392,7 @@ num_trues num_falses bitVector HL comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none avgTimeOnSite PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -281,6 +411,7 @@ num_trues num_falses bitVector HL comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none adRevenue PREHOOK: type: DESCTABLE PREHOOK: Input: default@uservisits_web_text_none @@ -299,6 +430,7 @@ num_trues num_falses bitVector HL comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: CREATE TEMPORARY TABLE empty_tab( a int, b double, @@ -383,7 +515,8 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-1 - Column Stats Work + Stats Work + Basic Stats Work: Column Stats Desc: Columns: a, b, c, d, e Column Types: int, double, string, boolean, binary @@ -392,10 +525,12 @@ STAGE PLANS: PREHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e PREHOOK: type: QUERY PREHOOK: Input: default@empty_tab +PREHOOK: Output: default@empty_tab #### A masked pattern was here #### POSTHOOK: query: analyze table empty_tab compute statistics for columns a,b,c,d,e POSTHOOK: type: QUERY POSTHOOK: Input: default@empty_tab +POSTHOOK: Output: default@empty_tab #### A masked pattern was here #### PREHOOK: query: desc formatted empty_tab a PREHOOK: type: DESCTABLE @@ -504,6 +639,7 @@ POSTHOOK: query: desc extended default.UserVisits_web_text_none sourceIP POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@uservisits_web_text_none sourceIP string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sourceIP PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -558,14 +694,17 @@ num_trues num_falses bitVector HL comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"adRevenue\":\"true\",\"avgTimeOnSite\":\"true\",\"sourceIP\":\"true\"}} PREHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword PREHOOK: type: QUERY PREHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +PREHOOK: Output: test@uservisits_web_text_none POSTHOOK: query: analyze table UserVisits_web_text_none compute statistics for columns sKeyword POSTHOOK: type: QUERY POSTHOOK: Input: test@uservisits_web_text_none #### A masked pattern was here #### +POSTHOOK: Output: test@uservisits_web_text_none PREHOOK: query: desc extended UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -573,6 +712,7 @@ POSTHOOK: query: desc extended UserVisits_web_text_none sKeyword POSTHOOK: type: DESCTABLE POSTHOOK: Input: test@uservisits_web_text_none sKeyword string from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"sKeyword\":\"true\"}} PREHOOK: query: desc formatted UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -591,6 +731,7 @@ num_trues num_falses bitVector HL comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"sKeyword\":\"true\"}} PREHOOK: query: desc formatted test.UserVisits_web_text_none sKeyword PREHOOK: type: DESCTABLE PREHOOK: Input: test@uservisits_web_text_none @@ -609,3 +750,4 @@ num_trues num_falses bitVector HL comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"sKeyword\":\"true\"}} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index 59437b80b4..a02a57c00b 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -77,7 +77,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.t @@ -121,7 +121,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.t"} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out index 546ae600ae..e5b1d74654 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out @@ -1129,7 +1129,7 @@ Reducer 8 <- Map 18 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 20 (SIMPLE_EDGE), Union 5 (CONTAINS) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -1324,13 +1324,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_72] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -1436,7 +1436,7 @@ Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) Reducer 9 <- Union 8 (SIMPLE_EDGE) Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.a"} @@ -1675,13 +1675,13 @@ Stage-5 table:{"name:":"default.c"} Please refer to the previous Group By Operator [GBY_112] Stage-6 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.b"} Please refer to the previous Stage-4 Stage-7 - Stats-Aggr Operator + Stats Work{} Stage-2 Move Operator table:{"name:":"default.c"} @@ -1744,7 +1744,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -1791,7 +1791,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT substr(_col1, 5))"],keys:_col0, _col1 Please refer to the previous Group By Operator [GBY_11] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -1976,7 +1976,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -2035,7 +2035,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} @@ -2094,7 +2094,7 @@ Reducer 4 <- Union 3 (SIMPLE_EDGE) Reducer 5 <- Union 3 (SIMPLE_EDGE) Stage-4 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.dest1"} @@ -2143,7 +2143,7 @@ Stage-4 Output:["_col0","_col1","_col2"],aggregations:["count(DISTINCT KEY._col2:0._col0)"],keys:KEY._col0, KEY._col1 <- Please refer to the previous Union 3 [SIMPLE_EDGE] Stage-5 - Stats-Aggr Operator + Stats Work{} Stage-1 Move Operator table:{"name:":"default.dest2"} diff --git ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 8ea8ac7984..12a13b25bc 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -225,7 +225,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze analyze table src_stats compute statistics POSTHOOK: type: QUERY Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Map 1 TableScan [TS_0] (rows=500/500 width=10) @@ -248,21 +248,19 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-3 - Column Stats Work{} - Stage-2 - Stats-Aggr Operator - Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=1248) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=350) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=350) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] +Stage-2 + Stats Work{} + Stage-0 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=1248) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500/500 width=350) + Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=350) + default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] PREHOOK: query: CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) PREHOOK: type: CREATEMACRO @@ -342,7 +340,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.src_autho_test @@ -653,7 +651,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.orc_merge5"} @@ -667,11 +665,11 @@ Stage-3 Map 1 File Output Operator [FS_3] table:{"name:":"default.orc_merge5"} - Select Operator [SEL_2] (rows=1/3 width=352) + Select Operator [SEL_2] (rows=306/3 width=335) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_4] (rows=1/3 width=352) + Filter Operator [FIL_4] (rows=306/3 width=335) predicate:(userid <= 13) - TableScan [TS_0] (rows=1/15000 width=352) + TableScan [TS_0] (rows=919/15000 width=335) default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] Stage-4(CONDITIONAL) File Merge diff --git ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index 7cdc5c9209..6fc3f52f20 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -27,7 +27,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze analyze table src_stats compute statistics POSTHOOK: type: QUERY Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Map 1 TableScan [TS_0] (rows=500/500 width=10) @@ -50,21 +50,19 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-3 - Column Stats Work{} - Stage-2 - Stats-Aggr Operator - Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=1248) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=350) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=350) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] +Stage-2 + Stats Work{} + Stage-0 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=1248) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500/500 width=350) + Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=350) + default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] PREHOOK: query: drop table src_multi2 PREHOOK: type: DROPTABLE @@ -101,66 +99,64 @@ Reducer 3 <- Union 2 (SIMPLE_EDGE) Reducer 4 <- Map 7 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE) -Stage-4 - Column Stats Work{} - Stage-3 - Stats-Aggr Operator - Stage-0 - Move Operator - table:{"name:":"default.src_multi2"} - Stage-2 - Dependency Collection{} - Stage-1 - Reducer 5 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] - File Output Operator [FS_19] - table:{"name:":"default.src_multi2"} - Select Operator [SEL_18] (rows=849/508 width=178) - Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_26] (rows=849/508 width=178) - Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col0","_col3"] - <-Map 7 [SIMPLE_EDGE] - SHUFFLE [RS_16] - PartitionCols:_col0 - Select Operator [SEL_14] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_25] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_12] (rows=500/500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 3 [SIMPLE_EDGE] - SHUFFLE [RS_15] - PartitionCols:_col0 - Select Operator [SEL_11] (rows=525/319 width=178) - Output:["_col0"] - Group By Operator [GBY_10] (rows=525/319 width=178) - Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 - <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_2] (rows=500/500 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_23] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - Output:["key","value"] - <-Map 6 [CONTAINS] - Reduce Output Operator [RS_9] - PartitionCols:_col0, _col1 - Select Operator [SEL_5] (rows=25/25 width=175) - Output:["_col0","_col1"] - Filter Operator [FIL_24] (rows=25/25 width=175) - predicate:key is not null - TableScan [TS_3] (rows=25/25 width=175) - Output:["key","value"] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=849/508 width=178) - Output:["key","value"] - Please refer to the previous Select Operator [SEL_18] +Stage-3 + Stats Work{} + Stage-0 + Move Operator + table:{"name:":"default.src_multi2"} + Stage-2 + Dependency Collection{} + Stage-1 + Reducer 5 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] + <-Reducer 4 [CUSTOM_SIMPLE_EDGE] + File Output Operator [FS_19] + table:{"name:":"default.src_multi2"} + Select Operator [SEL_18] (rows=849/508 width=178) + Output:["_col0","_col1"] + Merge Join Operator [MERGEJOIN_26] (rows=849/508 width=178) + Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col0","_col3"] + <-Map 7 [SIMPLE_EDGE] + SHUFFLE [RS_16] + PartitionCols:_col0 + Select Operator [SEL_14] (rows=500/500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_25] (rows=500/500 width=178) + predicate:key is not null + TableScan [TS_12] (rows=500/500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_15] + PartitionCols:_col0 + Select Operator [SEL_11] (rows=525/319 width=178) + Output:["_col0"] + Group By Operator [GBY_10] (rows=525/319 width=178) + Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 + <-Union 2 [SIMPLE_EDGE] + <-Map 1 [CONTAINS] + Reduce Output Operator [RS_9] + PartitionCols:_col0, _col1 + Select Operator [SEL_2] (rows=500/500 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_23] (rows=500/500 width=178) + predicate:key is not null + TableScan [TS_0] (rows=500/500 width=178) + Output:["key","value"] + <-Map 6 [CONTAINS] + Reduce Output Operator [RS_9] + PartitionCols:_col0, _col1 + Select Operator [SEL_5] (rows=25/25 width=175) + Output:["_col0","_col1"] + Filter Operator [FIL_24] (rows=25/25 width=175) + predicate:key is not null + TableScan [TS_3] (rows=25/25 width=175) + Output:["key","value"] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=849/508 width=178) + Output:["key","value"] + Please refer to the previous Select Operator [SEL_18] PREHOOK: query: select count(*) from (select * from src union select * from src1)subq PREHOOK: type: QUERY @@ -285,7 +281,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.acid_uami"} @@ -295,17 +291,17 @@ Stage-3 Reducer 2 File Output Operator [FS_8] table:{"name:":"default.acid_uami"} - Select Operator [SEL_4] (rows=1/2 width=302) + Select Operator [SEL_4] (rows=8/2 width=328) Output:["_col0","_col1","_col2","_col3"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_3] PartitionCols:UDFToInteger(_col0) - Select Operator [SEL_2] (rows=1/2 width=302) + Select Operator [SEL_2] (rows=8/2 width=328) Output:["_col0","_col1","_col3"] - Filter Operator [FIL_9] (rows=1/2 width=226) + Filter Operator [FIL_9] (rows=8/2 width=328) predicate:((de = 109.23) or (de = 119.23)) - TableScan [TS_0] (rows=1/4 width=226) - default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["i","de","vc"] + TableScan [TS_0] (rows=8/4 width=328) + default@acid_uami,acid_uami, ACID table,Tbl:COMPLETE,Col:NONE,Output:["i","de","vc"] PREHOOK: query: select * from acid_uami order by de PREHOOK: type: QUERY @@ -398,7 +394,7 @@ Vertex dependency in root stage Reducer 2 <- Map 1 (SIMPLE_EDGE) Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.acid_dot"} diff --git ql/src/test/results/clientpositive/tez/explainuser_3.q.out ql/src/test/results/clientpositive/tez/explainuser_3.q.out index cd17cfa3c7..c4f193ad2e 100644 --- ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -189,7 +189,7 @@ PREHOOK: type: QUERY POSTHOOK: query: explain analyze table src compute statistics POSTHOOK: type: QUERY Stage-2 - Stats-Aggr Operator + Stats Work{} Stage-0 Map 1 TableScan [TS_0] (rows=500 width=10) @@ -202,23 +202,21 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-3 - Column Stats Work{} - Stage-2 - Stats-Aggr Operator - Stage-0 - Reducer 2 - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=880) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] +Stage-2 + Stats Work{} + Stage-0 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_3] + Group By Operator [GBY_2] (rows=1 width=880) + Output:["_col0","_col1"],aggregations:["compute_stats(key, 'hll')","compute_stats(value, 'hll')"] + Select Operator [SEL_1] (rows=500 width=178) + Output:["key","value"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: explain CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) @@ -268,7 +266,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-4 Create Table Operator: name:default.src_autho_test @@ -497,7 +495,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Stage-3 - Stats-Aggr Operator + Stats Work{} Stage-0 Move Operator table:{"name:":"default.orc_merge5"} @@ -511,11 +509,11 @@ Stage-3 Map 1 vectorized File Output Operator [FS_8] table:{"name:":"default.orc_merge5"} - Select Operator [SEL_7] (rows=1 width=352) + Select Operator [SEL_7] (rows=306 width=335) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_6] (rows=1 width=352) + Filter Operator [FIL_6] (rows=306 width=335) predicate:(userid <= 13) - TableScan [TS_0] (rows=1 width=352) + TableScan [TS_0] (rows=919 width=335) default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"] Stage-4(CONDITIONAL) File Merge diff --git ql/src/test/results/clientpositive/truncate_column.q.out ql/src/test/results/clientpositive/truncate_column.q.out index 3a3ea8c9a4..067aaf3f0b 100644 --- ql/src/test/results/clientpositive/truncate_column.q.out +++ ql/src/test/results/clientpositive/truncate_column.q.out @@ -96,8 +96,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 10 - rawDataSize 94 totalSize 150 #### A masked pattern was here #### @@ -165,8 +163,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 10 - rawDataSize 94 totalSize 75 #### A masked pattern was here #### @@ -224,8 +220,6 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: numFiles 1 - numRows 10 - rawDataSize 94 totalSize 75 #### A masked pattern was here #### @@ -355,8 +349,6 @@ Table Type: MANAGED_TABLE Table Parameters: #### A masked pattern was here #### numFiles 1 - numRows 10 - rawDataSize 94 totalSize 150 #### A masked pattern was here #### @@ -415,8 +407,6 @@ Table Type: MANAGED_TABLE Table Parameters: #### A masked pattern was here #### numFiles 1 - numRows 10 - rawDataSize 94 totalSize 75 #### A masked pattern was here #### @@ -552,8 +542,6 @@ Table: test_tab_part #### A masked pattern was here #### Partition Parameters: numFiles 1 - numRows 10 - rawDataSize 94 totalSize 150 #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out index 61502c6bc1..9a462541d4 100644 --- ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out +++ ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out @@ -75,10 +75,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.test_tab numFiles 2 - numRows 500 partition_columns part partition_columns.types string - rawDataSize 4812 serialization.ddl struct test_tab { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -108,16 +106,16 @@ STAGE PLANS: Processor Tree: TableScan alias: test_tab - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1761 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key = '484') (type: boolean) - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '484' (type: string), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT * FROM test_tab WHERE part = '1' AND key = '484' @@ -157,10 +155,8 @@ STAGE PLANS: #### A masked pattern was here #### name default.test_tab numFiles 2 - numRows 500 partition_columns part partition_columns.types string - rawDataSize 4812 serialization.ddl struct test_tab { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -190,16 +186,16 @@ STAGE PLANS: Processor Tree: TableScan alias: test_tab - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 1761 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false predicate: (key = '0') (type: boolean) - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: '0' (type: string), value (type: string), '1' (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 880 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: SELECT * FROM test_tab WHERE part = '1' AND key = '0' diff --git ql/src/test/results/clientpositive/tunable_ndv.q.out ql/src/test/results/clientpositive/tunable_ndv.q.out index ba2a0ead84..a232eaaf09 100644 --- ql/src/test/results/clientpositive/tunable_ndv.q.out +++ ql/src/test/results/clientpositive/tunable_ndv.q.out @@ -60,12 +60,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d partition(year=2000) locid PREHOOK: type: DESCTABLE @@ -197,41 +203,57 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2000') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2000 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2001') compute statistics for columns state,locid POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2001 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_2d locid PREHOOK: type: DESCTABLE diff --git ql/src/test/results/clientpositive/udf1.q.out ql/src/test/results/clientpositive/udf1.q.out index eebd90f891..b59649be8b 100644 --- ql/src/test/results/clientpositive/udf1.q.out +++ ql/src/test/results/clientpositive/udf1.q.out @@ -85,7 +85,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf3.q.out ql/src/test/results/clientpositive/udf3.q.out index 96038f12af..e52a827862 100644 --- ql/src/test/results/clientpositive/udf3.q.out +++ ql/src/test/results/clientpositive/udf3.q.out @@ -67,7 +67,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), min(CAST('' AS INT)), max(CAST('' AS INT)) diff --git ql/src/test/results/clientpositive/udf_10_trims.q.out ql/src/test/results/clientpositive/udf_10_trims.q.out index 3a5303adfe..d15dbc219e 100644 --- ql/src/test/results/clientpositive/udf_10_trims.q.out +++ ql/src/test/results/clientpositive/udf_10_trims.q.out @@ -71,7 +71,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf_character_length.q.out ql/src/test/results/clientpositive/udf_character_length.q.out index 332ec95644..47ed6787ff 100644 --- ql/src/test/results/clientpositive/udf_character_length.q.out +++ ql/src/test/results/clientpositive/udf_character_length.q.out @@ -92,7 +92,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf_length.q.out ql/src/test/results/clientpositive/udf_length.q.out index fc795bbcf4..e21df147b2 100644 --- ql/src/test/results/clientpositive/udf_length.q.out +++ ql/src/test/results/clientpositive/udf_length.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf_octet_length.q.out ql/src/test/results/clientpositive/udf_octet_length.q.out index f8738f813c..4d742b1bc0 100644 --- ql/src/test/results/clientpositive/udf_octet_length.q.out +++ ql/src/test/results/clientpositive/udf_octet_length.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/udf_reverse.q.out ql/src/test/results/clientpositive/udf_reverse.q.out index 28b0c9f197..316780e89e 100644 --- ql/src/test/results/clientpositive/udf_reverse.q.out +++ ql/src/test/results/clientpositive/udf_reverse.q.out @@ -75,7 +75,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/union10.q.out ql/src/test/results/clientpositive/union10.q.out index e14e5e0dd2..f5e2b25801 100644 --- ql/src/test/results/clientpositive/union10.q.out +++ ql/src/test/results/clientpositive/union10.q.out @@ -139,7 +139,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union12.q.out ql/src/test/results/clientpositive/union12.q.out index 10540f9c49..a6cd633e3b 100644 --- ql/src/test/results/clientpositive/union12.q.out +++ ql/src/test/results/clientpositive/union12.q.out @@ -139,7 +139,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union17.q.out ql/src/test/results/clientpositive/union17.q.out index bff29f6f4e..d997c29444 100644 --- ql/src/test/results/clientpositive/union17.q.out +++ ql/src/test/results/clientpositive/union17.q.out @@ -164,7 +164,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -206,7 +207,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-6 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/union18.q.out ql/src/test/results/clientpositive/union18.q.out index 702ff103f9..5f737f68fa 100644 --- ql/src/test/results/clientpositive/union18.q.out +++ ql/src/test/results/clientpositive/union18.q.out @@ -157,7 +157,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -209,7 +210,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-11 Map Reduce diff --git ql/src/test/results/clientpositive/union19.q.out ql/src/test/results/clientpositive/union19.q.out index 35530912ff..e4ff0d5b74 100644 --- ql/src/test/results/clientpositive/union19.q.out +++ ql/src/test/results/clientpositive/union19.q.out @@ -165,7 +165,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -178,7 +179,8 @@ STAGE PLANS: name: default.dest2 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 UNION ALL diff --git ql/src/test/results/clientpositive/union22.q.out ql/src/test/results/clientpositive/union22.q.out index 9134bdf19f..e4b4a0914b 100644 --- ql/src/test/results/clientpositive/union22.q.out +++ ql/src/test/results/clientpositive/union22.q.out @@ -499,7 +499,8 @@ STAGE PLANS: name: default.dst_union22 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-4 diff --git ql/src/test/results/clientpositive/union25.q.out ql/src/test/results/clientpositive/union25.q.out index a287a97967..9eac674e7d 100644 --- ql/src/test/results/clientpositive/union25.q.out +++ ql/src/test/results/clientpositive/union25.q.out @@ -197,5 +197,6 @@ STAGE PLANS: name: default.tmp_unionall Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: diff --git ql/src/test/results/clientpositive/union28.q.out ql/src/test/results/clientpositive/union28.q.out index c3789d08e9..9b203f5461 100644 --- ql/src/test/results/clientpositive/union28.q.out +++ ql/src/test/results/clientpositive/union28.q.out @@ -153,7 +153,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union29.q.out ql/src/test/results/clientpositive/union29.q.out index 87ba275b87..b933426d3f 100644 --- ql/src/test/results/clientpositive/union29.q.out +++ ql/src/test/results/clientpositive/union29.q.out @@ -130,7 +130,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/union30.q.out ql/src/test/results/clientpositive/union30.q.out index 862ebc11d3..3ee716bffa 100644 --- ql/src/test/results/clientpositive/union30.q.out +++ ql/src/test/results/clientpositive/union30.q.out @@ -188,7 +188,8 @@ STAGE PLANS: name: default.union_subq_union Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union31.q.out ql/src/test/results/clientpositive/union31.q.out index b7a63fc6f0..4edaed63b8 100644 --- ql/src/test/results/clientpositive/union31.q.out +++ ql/src/test/results/clientpositive/union31.q.out @@ -193,7 +193,8 @@ STAGE PLANS: name: default.t3 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -236,7 +237,8 @@ STAGE PLANS: name: default.t4 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select * from t1 @@ -456,7 +458,8 @@ STAGE PLANS: name: default.t5 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -469,7 +472,8 @@ STAGE PLANS: name: default.t6 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-6 Map Reduce @@ -782,7 +786,8 @@ STAGE PLANS: name: default.t7 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -795,7 +800,8 @@ STAGE PLANS: name: default.t8 Stage: Stage-5 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from ( diff --git ql/src/test/results/clientpositive/union33.q.out ql/src/test/results/clientpositive/union33.q.out index 17aeecd254..2709d99784 100644 --- ql/src/test/results/clientpositive/union33.q.out +++ ql/src/test/results/clientpositive/union33.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.test_src Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce @@ -371,7 +372,8 @@ STAGE PLANS: name: default.test_src Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce diff --git ql/src/test/results/clientpositive/union4.q.out ql/src/test/results/clientpositive/union4.q.out index f3cd59c42e..cdaa78feb5 100644 --- ql/src/test/results/clientpositive/union4.q.out +++ ql/src/test/results/clientpositive/union4.q.out @@ -119,7 +119,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union6.q.out ql/src/test/results/clientpositive/union6.q.out index fc66cf1254..f7a578c654 100644 --- ql/src/test/results/clientpositive/union6.q.out +++ ql/src/test/results/clientpositive/union6.q.out @@ -116,7 +116,8 @@ STAGE PLANS: name: default.tmptable Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-4 Map Reduce diff --git ql/src/test/results/clientpositive/union_lateralview.q.out ql/src/test/results/clientpositive/union_lateralview.q.out index f563476de0..c45bf7580f 100644 --- ql/src/test/results/clientpositive/union_lateralview.q.out +++ ql/src/test/results/clientpositive/union_lateralview.q.out @@ -190,7 +190,8 @@ STAGE PLANS: name: default.test_union_lateral_view Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view SELECT b.key, d.arr_ele, d.value diff --git ql/src/test/results/clientpositive/union_stats.q.out ql/src/test/results/clientpositive/union_stats.q.out index ae0ef9ddf1..6585339adc 100644 --- ql/src/test/results/clientpositive/union_stats.q.out +++ ql/src/test/results/clientpositive/union_stats.q.out @@ -156,7 +156,8 @@ STAGE PLANS: name: default.t Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: #### A masked pattern was here #### Stage: Stage-3 diff --git ql/src/test/results/clientpositive/updateAccessTime.q.out ql/src/test/results/clientpositive/updateAccessTime.q.out index 777e84adc9..62e9f74aac 100644 --- ql/src/test/results/clientpositive/updateAccessTime.q.out +++ ql/src/test/results/clientpositive/updateAccessTime.q.out @@ -220,8 +220,10 @@ POSTHOOK: Output: default@src PREHOOK: query: ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value PREHOOK: type: QUERY PREHOOK: Input: default@src +PREHOOK: Output: default@src #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@src +POSTHOOK: Output: default@src #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/vector_bucket.q.out ql/src/test/results/clientpositive/vector_bucket.q.out index 3b74023c2b..057aac873c 100644 --- ql/src/test/results/clientpositive/vector_bucket.q.out +++ ql/src/test/results/clientpositive/vector_bucket.q.out @@ -87,7 +87,8 @@ STAGE PLANS: name: default.non_orc_table Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: select a, b from non_orc_table order by a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_char_4.q.out ql/src/test/results/clientpositive/vector_char_4.q.out index 1c58fd209b..b3ac2c0871 100644 --- ql/src/test/results/clientpositive/vector_char_4.q.out +++ ql/src/test/results/clientpositive/vector_char_4.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.char_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/vector_groupby4.q.out ql/src/test/results/clientpositive/vector_groupby4.q.out index 34b571e32d..1225a932f7 100644 --- ql/src/test/results/clientpositive/vector_groupby4.q.out +++ ql/src/test/results/clientpositive/vector_groupby4.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1) diff --git ql/src/test/results/clientpositive/vector_groupby6.q.out ql/src/test/results/clientpositive/vector_groupby6.q.out index bc86c15137..539fce1ec4 100644 --- ql/src/test/results/clientpositive/vector_groupby6.q.out +++ ql/src/test/results/clientpositive/vector_groupby6.q.out @@ -159,7 +159,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: FROM srcorc INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1) diff --git ql/src/test/results/clientpositive/vector_multi_insert.q.out ql/src/test/results/clientpositive/vector_multi_insert.q.out index 226eb56bbd..111ed64ae7 100644 --- ql/src/test/results/clientpositive/vector_multi_insert.q.out +++ ql/src/test/results/clientpositive/vector_multi_insert.q.out @@ -185,7 +185,8 @@ STAGE PLANS: name: default.orc_rn1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -237,7 +238,8 @@ STAGE PLANS: name: default.orc_rn2 Stage: Stage-10 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-11 Map Reduce @@ -289,7 +291,8 @@ STAGE PLANS: name: default.orc_rn3 Stage: Stage-16 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-17 Map Reduce diff --git ql/src/test/results/clientpositive/vector_outer_join1.q.out ql/src/test/results/clientpositive/vector_outer_join1.q.out index 70bce01c76..32d5581ce7 100644 --- ql/src/test/results/clientpositive/vector_outer_join1.q.out +++ ql/src/test/results/clientpositive/vector_outer_join1.q.out @@ -186,10 +186,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_join2.q.out ql/src/test/results/clientpositive/vector_outer_join2.q.out index 2265cb83bc..43f53ae320 100644 --- ql/src/test/results/clientpositive/vector_outer_join2.q.out +++ ql/src/test/results/clientpositive/vector_outer_join2.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_join3.q.out ql/src/test/results/clientpositive/vector_outer_join3.q.out index e4e482500f..d5c536c51e 100644 --- ql/src/test/results/clientpositive/vector_outer_join3.q.out +++ ql/src/test/results/clientpositive/vector_outer_join3.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_join4.q.out ql/src/test/results/clientpositive/vector_outer_join4.q.out index 125ec07353..7858f180b6 100644 --- ql/src/test/results/clientpositive/vector_outer_join4.q.out +++ ql/src/test/results/clientpositive/vector_outer_join4.q.out @@ -201,10 +201,12 @@ POSTHOOK: Output: default@small_alltypesorc_b PREHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_b +PREHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b +POSTHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_b PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out index a1a43b1059..660a437d45 100644 --- ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out +++ ql/src/test/results/clientpositive/vector_outer_reference_windowed.q.out @@ -95,28 +95,34 @@ c1 c2 PREHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_01 +PREHOOK: Output: default@e011_01 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_01 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_01 +POSTHOOK: Output: default@e011_01 #### A masked pattern was here #### _c0 _c1 PREHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_02 +PREHOOK: Output: default@e011_02 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_02 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_02 +POSTHOOK: Output: default@e011_02 #### A masked pattern was here #### _c0 _c1 PREHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@e011_03 +PREHOOK: Output: default@e011_03 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE e011_03 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@e011_03 +POSTHOOK: Output: default@e011_03 #### A masked pattern was here #### _c0 _c1 PREHOOK: query: explain vectorization detail @@ -141,7 +147,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1] @@ -152,7 +158,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) Group By Vectorization: @@ -330,7 +336,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1] @@ -341,7 +347,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 1] - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(c1) Group By Vectorization: @@ -356,7 +362,7 @@ STAGE PLANS: keys: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) sort order: ++ @@ -366,7 +372,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -398,7 +404,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -422,7 +428,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -446,7 +452,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -466,14 +472,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -535,19 +541,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -579,7 +585,7 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col0) Group By Vectorization: @@ -591,7 +597,7 @@ STAGE PLANS: keys: _col0 (type: decimal(15,2)), _col1 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -615,7 +621,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -647,7 +653,7 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -671,7 +677,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: decimal(25,2)) Execution mode: vectorized Map Vectorization: @@ -695,7 +701,7 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey1 (type: decimal(15,2)), KEY.reducesinkkey0 (type: decimal(15,2)), VALUE._col0 (type: decimal(25,2)) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -715,14 +721,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumHiveDecimal window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: decimal(35,2)) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -804,19 +810,19 @@ STAGE PLANS: value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Map Vectorization: enabled: false enabledConditionsNotMet: Vectorized map work only works with 1 TableScanOperator IS false @@ -1040,19 +1046,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: e011_01 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: c1 is not null (type: boolean) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: c1 (type: decimal(15,2)), c2 (type: decimal(15,2)) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: decimal(15,2)) sort order: + Map-reduce partition columns: _col0 (type: decimal(15,2)) - Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: decimal(15,2)) TableScan alias: e011_03 @@ -1085,7 +1091,7 @@ STAGE PLANS: 0 _col0 (type: decimal(15,2)) 1 _col0 (type: decimal(15,2)) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: corr(_col0, _col2) Group By Vectorization: @@ -1097,7 +1103,7 @@ STAGE PLANS: keys: _col1 (type: decimal(15,2)), _col3 (type: decimal(15,2)) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -1121,7 +1127,7 @@ STAGE PLANS: native: false nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false - Statistics: Num rows: 4 Data size: 39 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 13 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: struct) Execution mode: vectorized Map Vectorization: @@ -1153,11 +1159,11 @@ STAGE PLANS: keys: KEY._col0 (type: decimal(15,2)), KEY._col1 (type: decimal(15,2)) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: decimal(15,2)), _col0 (type: decimal(15,2)), _col2 (type: double) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator Function definitions: Input definition @@ -1177,14 +1183,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumDouble window frame: RANGE PRECEDING(MAX)~CURRENT - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: sum_window_0 (type: double) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 19 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git ql/src/test/results/clientpositive/vector_tablesample_rows.q.out ql/src/test/results/clientpositive/vector_tablesample_rows.q.out index 2d86d8c708..d7545d66f5 100644 --- ql/src/test/results/clientpositive/vector_tablesample_rows.q.out +++ ql/src/test/results/clientpositive/vector_tablesample_rows.q.out @@ -173,7 +173,8 @@ STAGE PLANS: name: default.decimal_2 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator @@ -381,7 +382,8 @@ STAGE PLANS: isTemporary: true Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce diff --git ql/src/test/results/clientpositive/vector_udf_character_length.q.out ql/src/test/results/clientpositive/vector_udf_character_length.q.out index 81d801c930..2960cd6e52 100644 --- ql/src/test/results/clientpositive/vector_udf_character_length.q.out +++ ql/src/test/results/clientpositive/vector_udf_character_length.q.out @@ -93,7 +93,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -251,8 +252,8 @@ POSTHOOK: query: SELECT character_length(dest2.name) FROM dest2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest2 #### A masked pattern was here #### -NULL 2 +NULL PREHOOK: query: EXPLAIN SELECT char_length(dest2.name) FROM dest2 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT char_length(dest2.name) FROM dest2 @@ -295,8 +296,8 @@ POSTHOOK: query: SELECT char_length(dest2.name) FROM dest2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest2 #### A masked pattern was here #### -NULL 2 +NULL PREHOOK: query: DROP TABLE dest1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@dest1 diff --git ql/src/test/results/clientpositive/vector_udf_octet_length.q.out ql/src/test/results/clientpositive/vector_udf_octet_length.q.out index c71cfef83f..46f5d654cb 100644 --- ql/src/test/results/clientpositive/vector_udf_octet_length.q.out +++ ql/src/test/results/clientpositive/vector_udf_octet_length.q.out @@ -76,7 +76,8 @@ STAGE PLANS: name: default.dest1 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Map Reduce @@ -234,8 +235,8 @@ POSTHOOK: query: SELECT octet_length(dest2.name) FROM dest2 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest2 #### A masked pattern was here #### -NULL 6 +NULL PREHOOK: query: DROP TABLE dest1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@dest1 diff --git ql/src/test/results/clientpositive/vector_varchar_4.q.out ql/src/test/results/clientpositive/vector_varchar_4.q.out index 205c67a6ae..d0f2065c97 100644 --- ql/src/test/results/clientpositive/vector_varchar_4.q.out +++ ql/src/test/results/clientpositive/vector_varchar_4.q.out @@ -201,7 +201,8 @@ STAGE PLANS: name: default.varchar_lazy_binary_columnar Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-3 Merge File Operator diff --git ql/src/test/results/clientpositive/vector_varchar_simple.q.out ql/src/test/results/clientpositive/vector_varchar_simple.q.out index 0f8bdb58c3..5d4c34439a 100644 --- ql/src/test/results/clientpositive/vector_varchar_simple.q.out +++ ql/src/test/results/clientpositive/vector_varchar_simple.q.out @@ -356,7 +356,8 @@ STAGE PLANS: name: default.varchar_3 Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: insert into table varchar_3 select cint from alltypesorc limit 10 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_windowing.q.out ql/src/test/results/clientpositive/vector_windowing.q.out index 12cd4ccbaa..e4fcd76ff1 100644 --- ql/src/test/results/clientpositive/vector_windowing.q.out +++ ql/src/test/results/clientpositive/vector_windowing.q.out @@ -5118,7 +5118,8 @@ STAGE PLANS: name: default.part_1 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-5 Map Reduce @@ -5345,7 +5346,8 @@ STAGE PLANS: name: default.part_2 Stage: Stage-8 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-9 Map Reduce @@ -5491,7 +5493,8 @@ STAGE PLANS: name: default.part_3 Stage: Stage-11 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from part INSERT OVERWRITE TABLE part_1 diff --git ql/src/test/results/clientpositive/vector_windowing_expressions.q.out ql/src/test/results/clientpositive/vector_windowing_expressions.q.out index 26e2f9b930..cb45c37197 100644 --- ql/src/test/results/clientpositive/vector_windowing_expressions.q.out +++ ql/src/test/results/clientpositive/vector_windowing_expressions.q.out @@ -1617,7 +1617,8 @@ STAGE PLANS: name: default.t1 Stage: Stage-3 - Stats-Aggr Operator + Stats Work + Basic Stats Work: Stage: Stage-1 Move Operator @@ -1630,7 +1631,8 @@ STAGE PLANS: name: default.t2 Stage: Stage-4 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: from (select sum(i) over (partition by ts order by i), s from over10k) tt insert overwrite table t1 select * insert overwrite table t2 select * PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/vector_windowing_streaming.q.out ql/src/test/results/clientpositive/vector_windowing_streaming.q.out index 658b105b50..aeeb937fa1 100644 --- ql/src/test/results/clientpositive/vector_windowing_streaming.q.out +++ ql/src/test/results/clientpositive/vector_windowing_streaming.q.out @@ -763,7 +763,8 @@ STAGE PLANS: name: default.sD Stage: Stage-2 - Stats-Aggr Operator + Stats Work + Basic Stats Work: PREHOOK: query: create table sD ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE as select * from (select ctinyint, cdouble, rank() over(partition by ctinyint order by cdouble) r from alltypesorc) a where r < 5 diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java index e2e3ada9e8..59190893e6 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@ -258,6 +258,14 @@ public static void setColumnStatsState(Map params, List } } + public static boolean canColumnStatsMerge(Map params, String colName) { + if (params == null) { + return false; + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + return stats.columnStats.containsKey(colName); + } + public static void clearColumnStatsState(Map params) { if (params == null) { return; @@ -295,7 +303,9 @@ public static void setStatsStateForCreateTable(Map params, } } setBasicStatsState(params, setting); - setColumnStatsState(params, cols); + if (TRUE.equals(setting)) { + setColumnStatsState(params, cols); + } } private static ColumnStatsAccurate parseStatsAcc(String statsAcc) { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 3deef0ee96..c203f827d9 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -7528,7 +7528,7 @@ public boolean updateTableColumnStatistics(ColumnStatistics colStats) MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics( ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj); writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); - colNames.add(statsObj.getColName()); + // There is no need to add colname again, otherwise we will get duplicate colNames. } // Set the table properties diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java index 1a2d38e556..64d07c77b7 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java @@ -70,7 +70,7 @@ public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsOb break; } default: - throw new IllegalArgumentException("Unknown stats type " + typeNew.toString()); + throw new IllegalArgumentException("Unknown stats type " + statsObjNew.getStatsData().getSetField()); } return agg; }