diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 9babc77..2a7627a 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -474,6 +474,8 @@ minillaplocal.query.files=acid_globallimit.q,\ cbo_rp_unionDistinct_2.q,\ cbo_rp_windowing_2.q,\ cbo_subq_not_in.q,\ + column_table_stats.q,\ + column_table_stats_orc.q,\ constprog_dpp.q,\ current_date_timestamp.q,\ correlationoptimizer1.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 3e749eb..36009bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1007,6 +1007,23 @@ public TableSpec(Hive db, HiveConf conf, String tableName, Map p } } + public TableSpec(Table tableHandle, List partitions) + throws HiveException { + this.tableHandle = tableHandle; + this.tableName = tableHandle.getTableName(); + if (partitions != null && !partitions.isEmpty()) { + this.specType = SpecType.STATIC_PARTITION; + this.partitions = partitions; + List partCols = this.tableHandle.getPartCols(); + this.partSpec = new LinkedHashMap<>(); + for (FieldSchema partCol : partCols) { + partSpec.put(partCol.getName(), null); + } + } else { + this.specType = SpecType.TABLE_ONLY; + } + } + public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartitionsSpec, boolean allowPartialPartitionsSpec) throws SemanticException { assert (ast.getToken().getType() == HiveParser.TOK_TAB @@ -1156,7 +1173,6 @@ public String toString() { private List colType; private boolean tblLvl; - public String getTableName() { return tableName; } @@ -1188,6 +1204,7 @@ public void setTblLvl(boolean isTblLvl) { public void setColType(List colType) { this.colType = colType; } + } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 7f5fdff..905431f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -199,6 +199,10 @@ protected void setupMapWork(MapWork mapWork, GenTezProcContext context, // All the setup is done in GenMapRedUtils GenMapRedUtils.setMapWork(mapWork, context.parseContext, context.inputs, partitions, root, alias, context.conf, false); + // we also collect table stats while collecting column stats. + if (context.parseContext.getAnalyzeRewrite() != null) { + mapWork.setGatheringStats(true); + } } // removes any union operator and clones the plan diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java index c13a404..46c24e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import java.io.Serializable; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.Stack; @@ -30,14 +32,17 @@ import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.StatsNoJobWork; import org.apache.hadoop.hive.ql.plan.StatsWork; @@ -65,9 +70,8 @@ public ProcessAnalyzeTable(GenTezUtils utils) { @SuppressWarnings("unchecked") @Override - public Object process(Node nd, Stack stack, - NodeProcessorCtx procContext, Object... nodeOutputs) - throws SemanticException { + public Object process(Node nd, Stack stack, NodeProcessorCtx procContext, + Object... nodeOutputs) throws SemanticException { GenTezProcContext context = (GenTezProcContext) procContext; @@ -79,18 +83,16 @@ public Object process(Node nd, Stack stack, if (parseContext.getQueryProperties().isAnalyzeCommand()) { - assert tableScan.getChildOperators() == null - || tableScan.getChildOperators().size() == 0; + assert tableScan.getChildOperators() == null || tableScan.getChildOperators().size() == 0; String alias = null; - for (String a: parseContext.getTopOps().keySet()) { + for (String a : parseContext.getTopOps().keySet()) { if (tableScan == parseContext.getTopOps().get(a)) { alias = a; } } assert alias != null; - TezWork tezWork = context.currentTask.getWork(); if (inputFormat.equals(OrcInputFormat.class)) { // For ORC, all the following statements are the same @@ -99,7 +101,8 @@ public Object process(Node nd, Stack stack, // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; // There will not be any Tez job above this task - StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec()); + StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata() + .getTableSpec()); snjWork.setStatsReliable(parseContext.getConf().getBoolVar( HiveConf.ConfVars.HIVE_STATS_RELIABLE)); // If partition is specified, get pruned partition list @@ -107,8 +110,8 @@ public Object process(Node nd, Stack stack, if (confirmedParts.size() > 0) { Table source = tableScan.getConf().getTableMetadata(); List partCols = GenMapRedUtils.getPartitionColumns(tableScan); - PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts, - partCols, false); + PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts, partCols, + false); snjWork.setPrunedPartitionList(partList); } Task snjTask = TaskFactory.get(snjWork, parseContext.getConf()); @@ -118,52 +121,101 @@ public Object process(Node nd, Stack stack, return true; } else { - // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS; - // The plan consists of a simple TezTask followed by a StatsTask. - // The Tez task is just a simple TableScanOperator + // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS; + // The plan consists of a simple TezTask followed by a StatsTask. + // The Tez task is just a simple TableScanOperator - StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); - statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); - statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); - statsWork.setSourceTask(context.currentTask); - statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE)); - Task statsTask = TaskFactory.get(statsWork, parseContext.getConf()); - context.currentTask.addDependentTask(statsTask); - - // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; - // The plan consists of a StatsTask only. - if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) { - statsTask.setParentTasks(null); - statsWork.setNoScanAnalyzeCommand(true); - context.rootTasks.remove(context.currentTask); - context.rootTasks.add(statsTask); - } + StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); + statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); + statsWork.setSourceTask(context.currentTask); + statsWork.setStatsReliable(parseContext.getConf().getBoolVar( + HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + Task statsTask = TaskFactory.get(statsWork, parseContext.getConf()); + context.currentTask.addDependentTask(statsTask); - // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan; - if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) { - handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask); - } + // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan; + // The plan consists of a StatsTask only. + if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) { + statsTask.setParentTasks(null); + statsWork.setNoScanAnalyzeCommand(true); + context.rootTasks.remove(context.currentTask); + context.rootTasks.add(statsTask); + } - // NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list, - // and pass it to setTaskPlan as the last parameter - Set confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan); - PrunedPartitionList partitions = null; - if (confirmedPartns.size() > 0) { - Table source = tableScan.getConf().getTableMetadata(); - List partCols = GenMapRedUtils.getPartitionColumns(tableScan); - partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false); - } + // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan; + if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) { + handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask); + } + + // NOTE: here we should use the new partition predicate pushdown API to + // get a list of pruned list, + // and pass it to setTaskPlan as the last parameter + Set confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan); + PrunedPartitionList partitions = null; + if (confirmedPartns.size() > 0) { + Table source = tableScan.getConf().getTableMetadata(); + List partCols = GenMapRedUtils.getPartitionColumns(tableScan); + partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false); + } - MapWork w = utils.createMapWork(context, tableScan, tezWork, partitions); - w.setGatheringStats(true); + MapWork w = utils.createMapWork(context, tableScan, tezWork, partitions); + w.setGatheringStats(true); - return true; + return true; + } + } else if (parseContext.getAnalyzeRewrite() != null) { + // we need to collect table stats while collecting column stats. + try { + context.currentTask.addDependentTask(genTableStats(context, tableScan)); + } catch (HiveException e) { + throw new SemanticException(e); } } return null; } + private Task genTableStats(GenTezProcContext context, TableScanOperator tableScan) + throws HiveException { + Class inputFormat = tableScan.getConf().getTableMetadata() + .getInputFormatClass(); + ParseContext parseContext = context.parseContext; + Table table = tableScan.getConf().getTableMetadata(); + List partitions = new ArrayList<>(); + if (table.isPartitioned()) { + partitions.addAll(parseContext.getPrunedPartitions(tableScan).getPartitions()); + for (Partition partn : partitions) { + LOG.debug("XXX: adding part: " + partn); + context.outputs.add(new WriteEntity(partn, WriteEntity.WriteType.DDL_NO_LOCK)); + } + } + TableSpec tableSpec = new TableSpec(table, partitions); + tableScan.getConf().getTableMetadata().setTableSpec(tableSpec); + + if (inputFormat.equals(OrcInputFormat.class)) { + // For ORC, there is no Tez Job for table stats. + StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata() + .getTableSpec()); + snjWork.setStatsReliable(parseContext.getConf().getBoolVar( + HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + // If partition is specified, get pruned partition list + if (partitions.size() > 0) { + snjWork.setPrunedPartitionList(parseContext.getPrunedPartitions(tableScan)); + } + return TaskFactory.get(snjWork, parseContext.getConf()); + } else { + + StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec()); + statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix()); + statsWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir()); + statsWork.setSourceTask(context.currentTask); + statsWork.setStatsReliable(parseContext.getConf().getBoolVar( + HiveConf.ConfVars.HIVE_STATS_RELIABLE)); + return TaskFactory.get(statsWork, parseContext.getConf()); + } + } + /** * handle partial scan command. * @@ -171,11 +223,12 @@ public Object process(Node nd, Stack stack, */ private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext, StatsWork statsWork, GenTezProcContext context, Task statsTask) - throws SemanticException { + throws SemanticException { String aggregationKey = tableScan.getConf().getStatsAggPrefix(); StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey); - List inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer); + List inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, + aggregationKeyBuffer); aggregationKey = aggregationKeyBuffer.toString(); // scan work diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index ee9101b..0732207 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -10260,7 +10260,11 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String RowResolver rwsch) throws SemanticException { - if (!qbp.isAnalyzeCommand()) { + // if it is not analyze command and not column stats, then do not gatherstats + // if it is column stats, but it is not tez, do not gatherstats + if ((!qbp.isAnalyzeCommand() && qbp.getAnalyzeRewrite() == null) + || (qbp.getAnalyzeRewrite() != null && !HiveConf.getVar(conf, + HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))) { tsDesc.setGatherStats(false); } else { if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { @@ -10283,15 +10287,6 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String tsDesc.addVirtualCols(vcList); String tblName = tab.getTableName(); - TableSpec tblSpec = qbp.getTableSpec(alias); - Map partSpec = tblSpec.getPartSpec(); - - if (partSpec != null) { - List cols = new ArrayList(); - cols.addAll(partSpec.keySet()); - tsDesc.setPartColumns(cols); - } - // Theoretically the key prefix could be any unique string shared // between TableScanOperator (when publishing) and StatsTask (when aggregating). // Here we use @@ -10300,13 +10295,27 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String // Currently, partition spec can only be static partition. String k = MetaStoreUtils.encodeTableName(tblName) + Path.SEPARATOR; tsDesc.setStatsAggPrefix(tab.getDbName()+"."+k); - + // set up WriteEntity for replication outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_SHARED)); // add WriteEntity for each matching partition if (tab.isPartitioned()) { - if (partSpec == null) { + List cols = new ArrayList(); + if (qbp.getAnalyzeRewrite() != null) { + List partitionCols = tab.getPartCols(); + for (FieldSchema fs : partitionCols) { + cols.add(fs.getName()); + } + tsDesc.setPartColumns(cols); + return; + } + TableSpec tblSpec = qbp.getTableSpec(alias); + Map partSpec = tblSpec.getPartSpec(); + if (partSpec != null) { + cols.addAll(partSpec.keySet()); + tsDesc.setPartColumns(cols); + } else { throw new SemanticException(ErrorMsg.NEED_PARTITION_SPECIFICATION.getMsg()); } List partitions = qbp.getTableSpec().partitions; diff --git a/ql/src/test/queries/clientpositive/column_table_stats.q b/ql/src/test/queries/clientpositive/column_table_stats.q new file mode 100644 index 0000000..991fa54 --- /dev/null +++ b/ql/src/test/queries/clientpositive/column_table_stats.q @@ -0,0 +1,88 @@ +set hive.mapred.mode=nonstrict; +-- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS s; + +CREATE TABLE s (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE s; + +desc formatted s; + +explain extended analyze table s compute statistics for columns; + +analyze table s compute statistics for columns; + +desc formatted s; + +DROP TABLE IF EXISTS spart; + +CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11"); + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12"); + + +desc formatted spart; + +explain extended analyze table spart compute statistics for columns; + +analyze table spart compute statistics for columns; + +desc formatted spart; + +desc formatted spart PARTITION(ds='2008-04-08', hr=11); +desc formatted spart PARTITION(ds='2008-04-08', hr=12); + +DROP TABLE IF EXISTS spart; + +CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11"); + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12"); + + +desc formatted spart; + +explain extended analyze table spart partition(ds,hr) compute statistics for columns; + +analyze table spart partition(ds,hr) compute statistics for columns; + +desc formatted spart; + +desc formatted spart PARTITION(ds='2008-04-08', hr=11); +desc formatted spart PARTITION(ds='2008-04-08', hr=12); + +DROP TABLE IF EXISTS spart; + +CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11"); + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12"); + + +desc formatted spart; + +explain extended analyze table spart partition(hr="11") compute statistics for columns; + +analyze table spart partition(hr="11") compute statistics for columns; + +desc formatted spart; + +desc formatted spart PARTITION(ds='2008-04-08', hr=11); +desc formatted spart PARTITION(ds='2008-04-08', hr=12); diff --git a/ql/src/test/queries/clientpositive/column_table_stats_orc.q b/ql/src/test/queries/clientpositive/column_table_stats_orc.q new file mode 100644 index 0000000..51fccd2 --- /dev/null +++ b/ql/src/test/queries/clientpositive/column_table_stats_orc.q @@ -0,0 +1,57 @@ +set hive.mapred.mode=nonstrict; +-- SORT_QUERY_RESULTS + +DROP TABLE IF EXISTS s; + +CREATE TABLE s (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS ORC; + +insert into table s values ('1','2'); + +desc formatted s; + +explain extended analyze table s compute statistics for columns; + +analyze table s compute statistics for columns; + +desc formatted s; + +DROP TABLE IF EXISTS spart; + +CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS ORC; + +insert into table spart PARTITION (ds="2008-04-08", hr="12") values ('1','2'); +insert into table spart PARTITION (ds="2008-04-08", hr="11") values ('1','2'); + +desc formatted spart; + +explain extended analyze table spart compute statistics for columns; + +analyze table spart compute statistics for columns; + +desc formatted spart; + +desc formatted spart PARTITION(ds='2008-04-08', hr=11); +desc formatted spart PARTITION(ds='2008-04-08', hr=12); + + +DROP TABLE IF EXISTS spart; + +CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS ORC; + +insert into table spart PARTITION (ds="2008-04-08", hr="12") values ('1','2'); +insert into table spart PARTITION (ds="2008-04-08", hr="11") values ('1','2'); + +desc formatted spart; + +explain extended analyze table spart partition(hr="11") compute statistics for columns; + +analyze table spart partition(hr="11") compute statistics for columns; + +desc formatted spart; + +desc formatted spart PARTITION(ds='2008-04-08', hr=11); +desc formatted spart PARTITION(ds='2008-04-08', hr=12); diff --git a/ql/src/test/results/clientpositive/llap/alter_table_invalidate_column_stats.q.out b/ql/src/test/results/clientpositive/llap/alter_table_invalidate_column_stats.q.out index c1c5f62..cf296e3 100644 --- a/ql/src/test/results/clientpositive/llap/alter_table_invalidate_column_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_table_invalidate_column_stats.q.out @@ -77,10 +77,12 @@ PREHOOK: query: analyze table testtable1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable1 POSTHOOK: query: analyze table testtable1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable1 PREHOOK: query: describe formatted statsdb1.testtable1 col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable1 @@ -202,12 +204,18 @@ PREHOOK: Input: statsdb1@testpart1 PREHOOK: Input: statsdb1@testpart1@part=part1 PREHOOK: Input: statsdb1@testpart1@part=part2 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part1 +PREHOOK: Output: statsdb1@testpart1@part=part2 POSTHOOK: query: analyze table testpart1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testpart1 POSTHOOK: Input: statsdb1@testpart1@part=part1 POSTHOOK: Input: statsdb1@testpart1@part=part2 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Output: statsdb1@testpart1@part=part2 PREHOOK: query: describe formatted statsdb1.testpart1 partition (part = 'part1') col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testpart1 @@ -552,10 +560,12 @@ PREHOOK: query: analyze table testtable1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testtable1 POSTHOOK: query: analyze table testtable1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testtable1 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testtable1 PREHOOK: query: describe formatted statsdb1.testtable1 col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testtable1 @@ -677,12 +687,18 @@ PREHOOK: Input: statsdb1@testpart1 PREHOOK: Input: statsdb1@testpart1@part=part1 PREHOOK: Input: statsdb1@testpart1@part=part2 #### A masked pattern was here #### +PREHOOK: Output: statsdb1@testpart1 +PREHOOK: Output: statsdb1@testpart1@part=part1 +PREHOOK: Output: statsdb1@testpart1@part=part2 POSTHOOK: query: analyze table testpart1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: statsdb1@testpart1 POSTHOOK: Input: statsdb1@testpart1@part=part1 POSTHOOK: Input: statsdb1@testpart1@part=part2 #### A masked pattern was here #### +POSTHOOK: Output: statsdb1@testpart1 +POSTHOOK: Output: statsdb1@testpart1@part=part1 +POSTHOOK: Output: statsdb1@testpart1@part=part2 PREHOOK: query: describe formatted statsdb1.testpart1 partition (part = 'part1') col1 PREHOOK: type: DESCTABLE PREHOOK: Input: statsdb1@testpart1 diff --git a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out index 80ccddd..6d941fd 100644 --- a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out +++ b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -189,10 +189,12 @@ POSTHOOK: Input: default@calendar PREHOOK: query: analyze table calendar compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@calendar +PREHOOK: Output: default@calendar #### A masked pattern was here #### POSTHOOK: query: analyze table calendar compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@calendar +POSTHOOK: Output: default@calendar #### A masked pattern was here #### PREHOOK: query: desc formatted calendar PREHOOK: type: DESCTABLE @@ -432,10 +434,12 @@ POSTHOOK: Input: default@calendar PREHOOK: query: analyze table calendar compute statistics for columns year PREHOOK: type: QUERY PREHOOK: Input: default@calendar +PREHOOK: Output: default@calendar #### A masked pattern was here #### POSTHOOK: query: analyze table calendar compute statistics for columns year POSTHOOK: type: QUERY POSTHOOK: Input: default@calendar +POSTHOOK: Output: default@calendar #### A masked pattern was here #### PREHOOK: query: desc formatted calendar PREHOOK: type: DESCTABLE @@ -565,10 +569,12 @@ POSTHOOK: Input: default@calendar PREHOOK: query: analyze table calendar compute statistics for columns month PREHOOK: type: QUERY PREHOOK: Input: default@calendar +PREHOOK: Output: default@calendar #### A masked pattern was here #### POSTHOOK: query: analyze table calendar compute statistics for columns month POSTHOOK: type: QUERY POSTHOOK: Input: default@calendar +POSTHOOK: Output: default@calendar #### A masked pattern was here #### PREHOOK: query: desc formatted calendar PREHOOK: type: DESCTABLE @@ -754,11 +760,15 @@ PREHOOK: query: analyze table calendarp partition (p=1) compute statistics for c PREHOOK: type: QUERY PREHOOK: Input: default@calendarp PREHOOK: Input: default@calendarp@p=1 +PREHOOK: Output: default@calendarp +PREHOOK: Output: default@calendarp@p=1 #### A masked pattern was here #### POSTHOOK: query: analyze table calendarp partition (p=1) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@calendarp POSTHOOK: Input: default@calendarp@p=1 +POSTHOOK: Output: default@calendarp +POSTHOOK: Output: default@calendarp@p=1 #### A masked pattern was here #### PREHOOK: query: desc formatted calendarp partition (p=1) PREHOOK: type: DESCTABLE diff --git a/ql/src/test/results/clientpositive/llap/column_table_stats.q.out b/ql/src/test/results/clientpositive/llap/column_table_stats.q.out new file mode 100644 index 0000000..b736477 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/column_table_stats.q.out @@ -0,0 +1,1421 @@ +PREHOOK: query: DROP TABLE IF EXISTS s +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS s +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE s (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s +POSTHOOK: query: CREATE TABLE s (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE s +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@s +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE s +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@s +PREHOOK: query: desc formatted s +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@s +POSTHOOK: query: desc formatted s +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@s +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table s compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table s compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Statistics Aggregation Key Prefix: default.s/ + GatherStats: true + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: struct), _col1 (type: struct) + auto parallelism: false + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: s + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.s + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct s { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.s + numFiles 1 + numRows 0 + rawDataSize 0 + serialization.ddl struct s { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.s + name: default.s + Truncated Path -> Alias: + /s [s] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: default.s/ + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.s + Is Table Level Stats: true + +PREHOOK: query: analyze table s compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@s +PREHOOK: Output: default@s +#### A masked pattern was here #### +POSTHOOK: query: analyze table s compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s +POSTHOOK: Output: default@s +#### A masked pattern was here #### +PREHOOK: query: desc formatted s +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@s +POSTHOOK: query: desc formatted s +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@s +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DROP TABLE IF EXISTS spart +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS spart +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@spart +POSTHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@spart +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@spart +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@spart +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numRows 0 + rawDataSize 0 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table spart compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table spart compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: spart + Statistics: Num rows: 58 Data size: 32968 Basic stats: COMPLETE Column stats: PARTIAL + Statistics Aggregation Key Prefix: default.spart/ + GatherStats: true + Select Operator + expressions: ds (type: string), hr (type: string), key (type: string), value (type: string) + outputColumnNames: ds, hr, key, value + Statistics: Num rows: 58 Data size: 32968 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + keys: ds (type: string), hr (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 2704 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2 Data size: 2704 Basic stats: COMPLETE Column stats: PARTIAL + tag: -1 + value expressions: _col2 (type: struct), _col3 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.spart + name: default.spart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.spart + name: default.spart + Truncated Path -> Alias: + /spart/ds=2008-04-08/hr=11 [spart] + /spart/ds=2008-04-08/hr=12 [spart] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 2656 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 2656 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 2656 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types struct:struct:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: default.spart/ + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.spart + Is Table Level Stats: false + +PREHOOK: query: analyze table spart compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@spart +PREHOOK: Input: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@spart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@spart +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@spart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: analyze table spart compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@spart +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DROP TABLE IF EXISTS spart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@spart +PREHOOK: Output: default@spart +POSTHOOK: query: DROP TABLE IF EXISTS spart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@spart +POSTHOOK: Output: default@spart +PREHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@spart +POSTHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@spart +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@spart +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@spart +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numRows 0 + rawDataSize 0 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table spart partition(ds,hr) compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table spart partition(ds,hr) compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: spart + Statistics: Num rows: 58 Data size: 32968 Basic stats: COMPLETE Column stats: PARTIAL + Statistics Aggregation Key Prefix: default.spart/ + GatherStats: true + Select Operator + expressions: ds (type: string), hr (type: string), key (type: string), value (type: string) + outputColumnNames: ds, hr, key, value + Statistics: Num rows: 58 Data size: 32968 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + keys: ds (type: string), hr (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 2704 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2 Data size: 2704 Basic stats: COMPLETE Column stats: PARTIAL + tag: -1 + value expressions: _col2 (type: struct), _col3 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.spart + name: default.spart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.spart + name: default.spart + Truncated Path -> Alias: + /spart/ds=2008-04-08/hr=11 [spart] + /spart/ds=2008-04-08/hr=12 [spart] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 2656 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 2656 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 2 Data size: 2656 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types struct:struct:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: default.spart/ + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.spart + Is Table Level Stats: false + +PREHOOK: query: analyze table spart partition(ds,hr) compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@spart +PREHOOK: Input: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@spart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@spart +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@spart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: analyze table spart partition(ds,hr) compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@spart +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 1000 + rawDataSize 10624 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DROP TABLE IF EXISTS spart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@spart +PREHOOK: Output: default@spart +POSTHOOK: query: DROP TABLE IF EXISTS spart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@spart +POSTHOOK: Output: default@spart +PREHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@spart +POSTHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@spart +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@spart +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@spart +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE spart PARTITION (ds="2008-04-08", hr="12") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numRows 0 + rawDataSize 0 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table spart partition(hr="11") compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table spart partition(hr="11") compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: spart + Statistics: Num rows: 29 Data size: 11148 Basic stats: COMPLETE Column stats: PARTIAL + Statistics Aggregation Key Prefix: default.spart/ + GatherStats: true + Select Operator + expressions: ds (type: string), key (type: string), value (type: string) + outputColumnNames: ds, key, value + Statistics: Num rows: 29 Data size: 11148 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + keys: ds (type: string), '11' (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1254 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), '11' (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), '11' (type: string) + Statistics: Num rows: 1 Data size: 1254 Basic stats: COMPLETE Column stats: PARTIAL + tag: -1 + value expressions: _col2 (type: struct), _col3 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: no inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 0 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 0 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.spart + name: default.spart + Truncated Path -> Alias: + /spart/ds=2008-04-08/hr=11 [spart] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), '11' (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1230 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), '11' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1230 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 1230 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types struct:struct:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + Stats Aggregation Key Prefix: default.spart/ + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.spart + Is Table Level Stats: false + +PREHOOK: query: analyze table spart partition(hr="11") compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@spart +PREHOOK: Input: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@spart +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: analyze table spart partition(hr="11") compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@spart +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + numFiles 2 + numRows 500 + rawDataSize 5312 + totalSize 11624 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 0 + rawDataSize 0 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out new file mode 100644 index 0000000..f038350 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out @@ -0,0 +1,969 @@ +PREHOOK: query: DROP TABLE IF EXISTS s +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS s +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE s (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s +POSTHOOK: query: CREATE TABLE s (key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s +PREHOOK: query: insert into table s values ('1','2') +PREHOOK: type: QUERY +PREHOOK: Output: default@s +POSTHOOK: query: insert into table s values ('1','2') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@s +POSTHOOK: Lineage: s.key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: s.value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: desc formatted s +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@s +POSTHOOK: query: desc formatted s +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@s +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 170 + totalSize 273 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table s compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table s compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s + Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE + Statistics Aggregation Key Prefix: default.s/ + GatherStats: true + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 1 Data size: 170 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col0 (type: struct), _col1 (type: struct) + auto parallelism: false + Execution mode: llap + LLAP IO: all inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: s + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.s + numFiles 1 + numRows 1 + rawDataSize 170 + serialization.ddl struct s { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 273 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.s + numFiles 1 + numRows 1 + rawDataSize 170 + serialization.ddl struct s { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 273 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.s + name: default.s + Truncated Path -> Alias: + /s [s] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 984 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:struct + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.s + Is Table Level Stats: true + +PREHOOK: query: analyze table s compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@s +PREHOOK: Output: default@s +#### A masked pattern was here #### +POSTHOOK: query: analyze table s compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s +POSTHOOK: Output: default@s +#### A masked pattern was here #### +PREHOOK: query: desc formatted s +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@s +POSTHOOK: query: desc formatted s +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@s +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 170 + totalSize 273 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DROP TABLE IF EXISTS spart +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS spart +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@spart +POSTHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@spart +PREHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="12") values ('1','2') +PREHOOK: type: QUERY +PREHOOK: Output: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="12") values ('1','2') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="11") values ('1','2') +PREHOOK: type: QUERY +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="11") values ('1','2') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table spart compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table spart compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: spart + Statistics: Num rows: 2 Data size: 1076 Basic stats: COMPLETE Column stats: PARTIAL + Statistics Aggregation Key Prefix: default.spart/ + GatherStats: true + Select Operator + expressions: ds (type: string), hr (type: string), key (type: string), value (type: string) + outputColumnNames: ds, hr, key, value + Statistics: Num rows: 2 Data size: 1076 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + keys: ds (type: string), hr (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 1 Data size: 1352 Basic stats: COMPLETE Column stats: PARTIAL + tag: -1 + value expressions: _col2 (type: struct), _col3 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: all inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 1 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 170 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 273 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.spart + name: default.spart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 1 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 170 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 273 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.spart + name: default.spart + Truncated Path -> Alias: + /spart/ds=2008-04-08/hr=11 [spart] + /spart/ds=2008-04-08/hr=12 [spart] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1328 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1328 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 1328 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types struct:struct:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.spart + Is Table Level Stats: false + +PREHOOK: query: analyze table spart compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@spart +PREHOOK: Input: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@spart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@spart +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@spart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: analyze table spart compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@spart +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 170 + totalSize 273 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 170 + totalSize 273 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: DROP TABLE IF EXISTS spart +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@spart +PREHOOK: Output: default@spart +POSTHOOK: query: DROP TABLE IF EXISTS spart +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@spart +POSTHOOK: Output: default@spart +PREHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@spart +POSTHOOK: query: CREATE TABLE spart (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@spart +PREHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="12") values ('1','2') +PREHOOK: type: QUERY +PREHOOK: Output: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="12") values ('1','2') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="11") values ('1','2') +PREHOOK: type: QUERY +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table spart PARTITION (ds="2008-04-08", hr="11") values ('1','2') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: spart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain extended analyze table spart partition(hr="11") compute statistics for columns +PREHOOK: type: QUERY +POSTHOOK: query: explain extended analyze table spart partition(hr="11") compute statistics for columns +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-0 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: spart + Statistics: Num rows: 1 Data size: 354 Basic stats: COMPLETE Column stats: PARTIAL + Statistics Aggregation Key Prefix: default.spart/ + GatherStats: true + Select Operator + expressions: ds (type: string), key (type: string), value (type: string) + outputColumnNames: ds, key, value + Statistics: Num rows: 1 Data size: 354 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: compute_stats(key, 16), compute_stats(value, 16) + keys: ds (type: string), '11' (type: string) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1254 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), '11' (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col0 (type: string), '11' (type: string) + Statistics: Num rows: 1 Data size: 1254 Basic stats: COMPLETE Column stats: PARTIAL + tag: -1 + value expressions: _col2 (type: struct), _col3 (type: struct) + auto parallelism: true + Execution mode: llap + LLAP IO: all inputs + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} + bucket_count -1 +#### A masked pattern was here #### + name default.spart + numFiles 1 + numRows 1 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 170 + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde + totalSize 273 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.spart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct spart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.spart + name: default.spart + Truncated Path -> Alias: + /spart/ds=2008-04-08/hr=11 [spart] + Reducer 2 + Execution mode: llap + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), '11' (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1230 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), '11' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 1230 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 1230 Basic stats: COMPLETE Column stats: PARTIAL +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types struct:struct:string:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Column Stats Work + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.spart + Is Table Level Stats: false + +PREHOOK: query: analyze table spart partition(hr="11") compute statistics for columns +PREHOOK: type: QUERY +PREHOOK: Input: default@spart +PREHOOK: Input: default@spart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@spart +PREHOOK: Output: default@spart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: analyze table spart partition(hr="11") compute statistics for columns +POSTHOOK: type: QUERY +POSTHOOK: Input: default@spart +POSTHOOK: Input: default@spart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@spart +POSTHOOK: Output: default@spart@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +PREHOOK: query: desc formatted spart +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=11) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + numFiles 1 + numRows 1 + rawDataSize 170 + totalSize 273 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@spart +POSTHOOK: query: desc formatted spart PARTITION(ds='2008-04-08', hr=12) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@spart +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 12] +Database: default +Table: spart +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 170 + totalSize 273 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 diff --git a/ql/src/test/results/clientpositive/llap/columnstats_part_coltype.q.out b/ql/src/test/results/clientpositive/llap/columnstats_part_coltype.q.out index 3e28e58..dc50fb7 100644 --- a/ql/src/test/results/clientpositive/llap/columnstats_part_coltype.q.out +++ b/ql/src/test/results/clientpositive/llap/columnstats_part_coltype.q.out @@ -64,11 +64,15 @@ PREHOOK: query: analyze table partcolstats partition (ds=date '2015-04-02', hr=2 PREHOOK: type: QUERY PREHOOK: Input: default@partcolstats PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partA +PREHOOK: Output: default@partcolstats +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstats partition (ds=date '2015-04-02', hr=2, part='partA') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcolstats POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partA +POSTHOOK: Output: default@partcolstats +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstats partition (ds=date '2015-04-02', hr=2, part='partA') key PREHOOK: type: DESCTABLE @@ -111,12 +115,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@partcolstats PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partA PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partB +PREHOOK: Output: default@partcolstats +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partB #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstats partition (ds=date '2015-04-02', hr=2, part) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcolstats POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partA POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partB +POSTHOOK: Output: default@partcolstats +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partB #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstats partition (ds=date '2015-04-02', hr=2, part='partB') key PREHOOK: type: DESCTABLE @@ -160,6 +170,10 @@ PREHOOK: Input: default@partcolstats PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partA PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partB PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=3/part=partA +PREHOOK: Output: default@partcolstats +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partB +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=3/part=partA #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstats partition (ds=date '2015-04-02', hr, part) compute statistics for columns POSTHOOK: type: QUERY @@ -167,6 +181,10 @@ POSTHOOK: Input: default@partcolstats POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partA POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partB POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=3/part=partA +POSTHOOK: Output: default@partcolstats +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partB +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=3/part=partA #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstats partition (ds=date '2015-04-02', hr=3, part='partA') key PREHOOK: type: DESCTABLE @@ -230,6 +248,12 @@ PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partB PREHOOK: Input: default@partcolstats@ds=2015-04-02/hr=3/part=partA PREHOOK: Input: default@partcolstats@ds=2015-04-03/hr=3/part=partA PREHOOK: Input: default@partcolstats@ds=2015-04-03/hr=3/part=partB +PREHOOK: Output: default@partcolstats +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partB +PREHOOK: Output: default@partcolstats@ds=2015-04-02/hr=3/part=partA +PREHOOK: Output: default@partcolstats@ds=2015-04-03/hr=3/part=partA +PREHOOK: Output: default@partcolstats@ds=2015-04-03/hr=3/part=partB #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstats partition (ds, hr, part) compute statistics for columns POSTHOOK: type: QUERY @@ -239,6 +263,12 @@ POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=2/part=partB POSTHOOK: Input: default@partcolstats@ds=2015-04-02/hr=3/part=partA POSTHOOK: Input: default@partcolstats@ds=2015-04-03/hr=3/part=partA POSTHOOK: Input: default@partcolstats@ds=2015-04-03/hr=3/part=partB +POSTHOOK: Output: default@partcolstats +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partA +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=2/part=partB +POSTHOOK: Output: default@partcolstats@ds=2015-04-02/hr=3/part=partA +POSTHOOK: Output: default@partcolstats@ds=2015-04-03/hr=3/part=partA +POSTHOOK: Output: default@partcolstats@ds=2015-04-03/hr=3/part=partB #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstats partition (ds=date '2015-04-03', hr=3, part='partA') key PREHOOK: type: DESCTABLE @@ -310,11 +340,15 @@ PREHOOK: query: analyze table partcolstatsnum partition (tint=100, sint=1000, bi PREHOOK: type: QUERY PREHOOK: Input: default@partcolstatsnum PREHOOK: Input: default@partcolstatsnum@tint=100/sint=1000/bint=1000000 +PREHOOK: Output: default@partcolstatsnum +PREHOOK: Output: default@partcolstatsnum@tint=100/sint=1000/bint=1000000 #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstatsnum partition (tint=100, sint=1000, bint=1000000) compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcolstatsnum POSTHOOK: Input: default@partcolstatsnum@tint=100/sint=1000/bint=1000000 +POSTHOOK: Output: default@partcolstatsnum +POSTHOOK: Output: default@partcolstatsnum@tint=100/sint=1000/bint=1000000 #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstatsnum partition (tint=100, sint=1000, bint=1000000) value PREHOOK: type: DESCTABLE @@ -359,11 +393,15 @@ PREHOOK: query: analyze table partcolstatsdec partition (decpart='1000.0001') co PREHOOK: type: QUERY PREHOOK: Input: default@partcolstatsdec PREHOOK: Input: default@partcolstatsdec@decpart=1000.0001 +PREHOOK: Output: default@partcolstatsdec +PREHOOK: Output: default@partcolstatsdec@decpart=1000.0001 #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstatsdec partition (decpart='1000.0001') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcolstatsdec POSTHOOK: Input: default@partcolstatsdec@decpart=1000.0001 +POSTHOOK: Output: default@partcolstatsdec +POSTHOOK: Output: default@partcolstatsdec@decpart=1000.0001 #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstatsdec partition (decpart='1000.0001') value PREHOOK: type: DESCTABLE @@ -408,11 +446,15 @@ PREHOOK: query: analyze table partcolstatschar partition (varpart='part1', charp PREHOOK: type: QUERY PREHOOK: Input: default@partcolstatschar PREHOOK: Input: default@partcolstatschar@varpart=part1/charpart=aaa +PREHOOK: Output: default@partcolstatschar +PREHOOK: Output: default@partcolstatschar@varpart=part1/charpart=aaa #### A masked pattern was here #### POSTHOOK: query: analyze table partcolstatschar partition (varpart='part1', charpart='aaa') compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partcolstatschar POSTHOOK: Input: default@partcolstatschar@varpart=part1/charpart=aaa +POSTHOOK: Output: default@partcolstatschar +POSTHOOK: Output: default@partcolstatschar@varpart=part1/charpart=aaa #### A masked pattern was here #### PREHOOK: query: describe formatted partcolstatschar partition (varpart='part1', charpart='aaa') value PREHOOK: type: DESCTABLE diff --git a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index ba14835..98ba6af 100644 --- a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -75,18 +75,22 @@ amount decimal(10,3) PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 +PREHOOK: Output: default@testdeci2 #### A masked pattern was here #### POSTHOOK: query: analyze table testdeci2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 +POSTHOOK: Output: default@testdeci2 #### A masked pattern was here #### PREHOOK: query: analyze table testdeci2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@testdeci2 +PREHOOK: Output: default@testdeci2 #### A masked pattern was here #### POSTHOOK: query: analyze table testdeci2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@testdeci2 +POSTHOOK: Output: default@testdeci2 #### A masked pattern was here #### PREHOOK: query: explain select s.id, diff --git a/ql/src/test/results/clientpositive/llap/drop_partition_with_stats.q.out b/ql/src/test/results/clientpositive/llap/drop_partition_with_stats.q.out index c6ab40d..2d68334 100644 --- a/ql/src/test/results/clientpositive/llap/drop_partition_with_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/drop_partition_with_stats.q.out @@ -42,22 +42,32 @@ PREHOOK: Input: partstatsdb1@testtable PREHOOK: Input: partstatsdb1@testtable@part1=p11/part2=P12 PREHOOK: Input: partstatsdb1@testtable@part1=p21/part2=P22 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable +PREHOOK: Output: partstatsdb1@testtable@part1=p11/part2=P12 +PREHOOK: Output: partstatsdb1@testtable@part1=p21/part2=P22 POSTHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable POSTHOOK: Input: partstatsdb1@testtable@part1=p11/part2=P12 POSTHOOK: Input: partstatsdb1@testtable@part1=p21/part2=P22 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable +POSTHOOK: Output: partstatsdb1@testtable@part1=p11/part2=P12 +POSTHOOK: Output: partstatsdb1@testtable@part1=p21/part2=P22 PREHOOK: query: ANALYZE TABLE testtable PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb1@testtable PREHOOK: Input: partstatsdb1@testtable@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable +PREHOOK: Output: partstatsdb1@testtable@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE testtable PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable POSTHOOK: Input: partstatsdb1@testtable@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable +POSTHOOK: Output: partstatsdb1@testtable@part1=p11/part2=P12 PREHOOK: query: CREATE TABLE IF NOT EXISTS TestTable1 (key STRING, value STRING) PARTITIONED BY (part1 STRING, Part2 STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:partstatsdb1 @@ -110,6 +120,11 @@ PREHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P12 PREHOOK: Input: partstatsdb1@testtable1@part1=p21/part2=P22 PREHOOK: Input: partstatsdb1@testtable1@part1=p31/part2=P32 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable1 +PREHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P11 +PREHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P12 +PREHOOK: Output: partstatsdb1@testtable1@part1=p21/part2=P22 +PREHOOK: Output: partstatsdb1@testtable1@part1=p31/part2=P32 POSTHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable1 @@ -118,28 +133,43 @@ POSTHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P12 POSTHOOK: Input: partstatsdb1@testtable1@part1=p21/part2=P22 POSTHOOK: Input: partstatsdb1@testtable1@part1=p31/part2=P32 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable1 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P11 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P12 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p21/part2=P22 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p31/part2=P32 PREHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb1@testtable1 PREHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P11 PREHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable1 +PREHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P11 +PREHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable1 POSTHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P11 POSTHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable1 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P11 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P12 PREHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb1@testtable1 PREHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable1 +PREHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable1 POSTHOOK: Input: partstatsdb1@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable1 +POSTHOOK: Output: partstatsdb1@testtable1@part1=p11/part2=P12 PREHOOK: query: CREATE TABLE IF NOT EXISTS TESTTABLE2 (key STRING, value STRING) PARTITIONED BY (part1 STRING, Part2 STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: database:partstatsdb1 @@ -172,22 +202,32 @@ PREHOOK: Input: partstatsdb1@testtable2 PREHOOK: Input: partstatsdb1@testtable2@part1=p11/part2=P12 PREHOOK: Input: partstatsdb1@testtable2@part1=p21/part2=P22 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable2 +PREHOOK: Output: partstatsdb1@testtable2@part1=p11/part2=P12 +PREHOOK: Output: partstatsdb1@testtable2@part1=p21/part2=P22 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable2 POSTHOOK: Input: partstatsdb1@testtable2@part1=p11/part2=P12 POSTHOOK: Input: partstatsdb1@testtable2@part1=p21/part2=P22 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable2 +POSTHOOK: Output: partstatsdb1@testtable2@part1=p11/part2=P12 +POSTHOOK: Output: partstatsdb1@testtable2@part1=p21/part2=P22 PREHOOK: query: ANALYZE TABLE TESTTABLE2 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb1@testtable2 PREHOOK: Input: partstatsdb1@testtable2@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb1@testtable2 +PREHOOK: Output: partstatsdb1@testtable2@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb1@testtable2 POSTHOOK: Input: partstatsdb1@testtable2@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb1@testtable2 +POSTHOOK: Output: partstatsdb1@testtable2@part1=p11/part2=P12 PREHOOK: query: ALTER TABLE partstatsdb1.testtable DROP PARTITION (part1='p11', Part2='P12') PREHOOK: type: ALTERTABLE_DROPPARTS PREHOOK: Input: partstatsdb1@testtable @@ -288,22 +328,32 @@ PREHOOK: Input: partstatsdb2@testtable PREHOOK: Input: partstatsdb2@testtable@part1=p11/part2=P12 PREHOOK: Input: partstatsdb2@testtable@part1=p21/part2=P22 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable +PREHOOK: Output: partstatsdb2@testtable@part1=p11/part2=P12 +PREHOOK: Output: partstatsdb2@testtable@part1=p21/part2=P22 POSTHOOK: query: ANALYZE TABLE testtable COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable POSTHOOK: Input: partstatsdb2@testtable@part1=p11/part2=P12 POSTHOOK: Input: partstatsdb2@testtable@part1=p21/part2=P22 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable +POSTHOOK: Output: partstatsdb2@testtable@part1=p11/part2=P12 +POSTHOOK: Output: partstatsdb2@testtable@part1=p21/part2=P22 PREHOOK: query: ANALYZE TABLE testtable PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb2@testtable PREHOOK: Input: partstatsdb2@testtable@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable +PREHOOK: Output: partstatsdb2@testtable@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE testtable PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable POSTHOOK: Input: partstatsdb2@testtable@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable +POSTHOOK: Output: partstatsdb2@testtable@part1=p11/part2=P12 PREHOOK: query: CREATE TABLE IF NOT EXISTS TestTable1 (key STRING, value STRING) PARTITIONED BY (part1 STRING, Part2 STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: PARTSTATSDB2@TestTable1 @@ -356,6 +406,11 @@ PREHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P12 PREHOOK: Input: partstatsdb2@testtable1@part1=p21/part2=P22 PREHOOK: Input: partstatsdb2@testtable1@part1=p31/part2=P32 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable1 +PREHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P11 +PREHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P12 +PREHOOK: Output: partstatsdb2@testtable1@part1=p21/part2=P22 +PREHOOK: Output: partstatsdb2@testtable1@part1=p31/part2=P32 POSTHOOK: query: ANALYZE TABLE TestTable1 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable1 @@ -364,28 +419,43 @@ POSTHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P12 POSTHOOK: Input: partstatsdb2@testtable1@part1=p21/part2=P22 POSTHOOK: Input: partstatsdb2@testtable1@part1=p31/part2=P32 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable1 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P11 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P12 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p21/part2=P22 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p31/part2=P32 PREHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb2@testtable1 PREHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P11 PREHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable1 +PREHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P11 +PREHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable1 POSTHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P11 POSTHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable1 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P11 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P12 PREHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb2@testtable1 PREHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable1 +PREHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE TestTable1 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable1 POSTHOOK: Input: partstatsdb2@testtable1@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable1 +POSTHOOK: Output: partstatsdb2@testtable1@part1=p11/part2=P12 PREHOOK: query: CREATE TABLE IF NOT EXISTS TESTTABLE2 (key STRING, value STRING) PARTITIONED BY (part1 STRING, Part2 STRING) PREHOOK: type: CREATETABLE PREHOOK: Output: PARTSTATSDB2@TESTTABLE2 @@ -418,22 +488,32 @@ PREHOOK: Input: partstatsdb2@testtable2 PREHOOK: Input: partstatsdb2@testtable2@part1=p11/part2=P12 PREHOOK: Input: partstatsdb2@testtable2@part1=p21/part2=P22 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable2 +PREHOOK: Output: partstatsdb2@testtable2@part1=p11/part2=P12 +PREHOOK: Output: partstatsdb2@testtable2@part1=p21/part2=P22 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable2 POSTHOOK: Input: partstatsdb2@testtable2@part1=p11/part2=P12 POSTHOOK: Input: partstatsdb2@testtable2@part1=p21/part2=P22 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable2 +POSTHOOK: Output: partstatsdb2@testtable2@part1=p11/part2=P12 +POSTHOOK: Output: partstatsdb2@testtable2@part1=p21/part2=P22 PREHOOK: query: ANALYZE TABLE TESTTABLE2 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key PREHOOK: type: QUERY PREHOOK: Input: partstatsdb2@testtable2 PREHOOK: Input: partstatsdb2@testtable2@part1=p11/part2=P12 #### A masked pattern was here #### +PREHOOK: Output: partstatsdb2@testtable2 +PREHOOK: Output: partstatsdb2@testtable2@part1=p11/part2=P12 POSTHOOK: query: ANALYZE TABLE TESTTABLE2 PARTITION (part1='p11', Part2='P12') COMPUTE STATISTICS FOR COLUMNS key POSTHOOK: type: QUERY POSTHOOK: Input: partstatsdb2@testtable2 POSTHOOK: Input: partstatsdb2@testtable2@part1=p11/part2=P12 #### A masked pattern was here #### +POSTHOOK: Output: partstatsdb2@testtable2 +POSTHOOK: Output: partstatsdb2@testtable2@part1=p11/part2=P12 PREHOOK: query: ALTER TABLE PARTSTATSDB2.testtable DROP PARTITION (part1='p11', Part2='P12') PREHOOK: type: ALTERTABLE_DROPPARTS PREHOOK: Input: partstatsdb2@testtable diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index e898111..087a531 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -127,10 +127,12 @@ POSTHOOK: Output: default@ss PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 PREHOOK: type: QUERY PREHOOK: Input: default@ss +PREHOOK: Output: default@ss #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 POSTHOOK: type: QUERY POSTHOOK: Input: default@ss +POSTHOOK: Output: default@ss #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS PREHOOK: type: QUERY @@ -143,10 +145,12 @@ POSTHOOK: Output: default@sr PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 PREHOOK: type: QUERY PREHOOK: Input: default@sr +PREHOOK: Output: default@sr #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 POSTHOOK: type: QUERY POSTHOOK: Input: default@sr +POSTHOOK: Output: default@sr #### A masked pattern was here #### PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS PREHOOK: type: QUERY @@ -159,10 +163,12 @@ POSTHOOK: Output: default@cs PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 PREHOOK: type: QUERY PREHOOK: Input: default@cs +PREHOOK: Output: default@cs #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 POSTHOOK: type: QUERY POSTHOOK: Input: default@cs +POSTHOOK: Output: default@cs #### A masked pattern was here #### PREHOOK: query: EXPLAIN SELECT x.key, z.value, y.value diff --git a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out index c9b8cfd..d97223c 100644 --- a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out +++ b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out @@ -83,21 +83,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2001 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2001') compute statistics for columns state,locid,cnt,zip POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2001 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2002') compute statistics for columns state,locid,cnt,zip PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2002 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2002 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2002') compute statistics for columns state,locid,cnt,zip POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2002 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2002 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2001') state PREHOOK: type: DESCTABLE @@ -376,21 +384,29 @@ PREHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2000 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2000') compute statistics for columns state,locid,cnt,zip POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2000 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2000 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_1d partition(year='2003') compute statistics for columns state,locid,cnt,zip PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_1d PREHOOK: Input: default@loc_orc_1d@year=2003 +PREHOOK: Output: default@loc_orc_1d +PREHOOK: Output: default@loc_orc_1d@year=2003 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_1d partition(year='2003') compute statistics for columns state,locid,cnt,zip POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_1d POSTHOOK: Input: default@loc_orc_1d@year=2003 +POSTHOOK: Output: default@loc_orc_1d +POSTHOOK: Output: default@loc_orc_1d@year=2003 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2000') state PREHOOK: type: DESCTABLE @@ -740,21 +756,29 @@ PREHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compu PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94086, year='2001') compute statistics for columns state,locid,cnt POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94086/year=2001 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94086/year=2001 #### A masked pattern was here #### PREHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2002') compute statistics for columns state,locid,cnt PREHOOK: type: QUERY PREHOOK: Input: default@loc_orc_2d PREHOOK: Input: default@loc_orc_2d@zip=94087/year=2002 +PREHOOK: Output: default@loc_orc_2d +PREHOOK: Output: default@loc_orc_2d@zip=94087/year=2002 #### A masked pattern was here #### POSTHOOK: query: analyze table loc_orc_2d partition(zip=94087, year='2002') compute statistics for columns state,locid,cnt POSTHOOK: type: QUERY POSTHOOK: Input: default@loc_orc_2d POSTHOOK: Input: default@loc_orc_2d@zip=94087/year=2002 +POSTHOOK: Output: default@loc_orc_2d +POSTHOOK: Output: default@loc_orc_2d@zip=94087/year=2002 #### A masked pattern was here #### PREHOOK: query: describe formatted loc_orc_2d partition(zip=94086, year='2001') state PREHOOK: type: DESCTABLE diff --git a/ql/src/test/results/clientpositive/llap/llap_stats.q.out b/ql/src/test/results/clientpositive/llap/llap_stats.q.out index f6921f1..3c393fc 100644 --- a/ql/src/test/results/clientpositive/llap/llap_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_stats.q.out @@ -94,6 +94,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-0 is a root stage Stage-2 depends on stages: Stage-0 + Stage-3 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-0 @@ -148,6 +149,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 Column Stats Work Column Stats Desc: Columns: ctinyint, csmallint @@ -167,6 +171,17 @@ PREHOOK: Input: default@llap_stats@cint=-9566 PREHOOK: Input: default@llap_stats@cint=15007 PREHOOK: Input: default@llap_stats@cint=4963 PREHOOK: Input: default@llap_stats@cint=7021 +PREHOOK: Output: default@llap_stats +PREHOOK: Output: default@llap_stats@cint=-13326 +PREHOOK: Output: default@llap_stats@cint=-15431 +PREHOOK: Output: default@llap_stats@cint=-15549 +PREHOOK: Output: default@llap_stats@cint=-15813 +PREHOOK: Output: default@llap_stats@cint=-4213 +PREHOOK: Output: default@llap_stats@cint=-7824 +PREHOOK: Output: default@llap_stats@cint=-9566 +PREHOOK: Output: default@llap_stats@cint=15007 +PREHOOK: Output: default@llap_stats@cint=4963 +PREHOOK: Output: default@llap_stats@cint=7021 #### A masked pattern was here #### POSTHOOK: query: analyze table llap_stats partition (cint) compute statistics for columns POSTHOOK: type: QUERY @@ -181,6 +196,17 @@ POSTHOOK: Input: default@llap_stats@cint=-9566 POSTHOOK: Input: default@llap_stats@cint=15007 POSTHOOK: Input: default@llap_stats@cint=4963 POSTHOOK: Input: default@llap_stats@cint=7021 +POSTHOOK: Output: default@llap_stats +POSTHOOK: Output: default@llap_stats@cint=-13326 +POSTHOOK: Output: default@llap_stats@cint=-15431 +POSTHOOK: Output: default@llap_stats@cint=-15549 +POSTHOOK: Output: default@llap_stats@cint=-15813 +POSTHOOK: Output: default@llap_stats@cint=-4213 +POSTHOOK: Output: default@llap_stats@cint=-7824 +POSTHOOK: Output: default@llap_stats@cint=-9566 +POSTHOOK: Output: default@llap_stats@cint=15007 +POSTHOOK: Output: default@llap_stats@cint=4963 +POSTHOOK: Output: default@llap_stats@cint=7021 #### A masked pattern was here #### PREHOOK: query: DROP TABLE llap_stats PREHOOK: type: DROPTABLE diff --git a/ql/src/test/results/clientpositive/llap/llapdecider.q.out b/ql/src/test/results/clientpositive/llap/llapdecider.q.out index d514f42..69312cd 100644 --- a/ql/src/test/results/clientpositive/llap/llapdecider.q.out +++ b/ql/src/test/results/clientpositive/llap/llapdecider.q.out @@ -230,10 +230,12 @@ STAGE PLANS: PREHOOK: query: analyze table src_orc compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@src_orc +PREHOOK: Output: default@src_orc #### A masked pattern was here #### POSTHOOK: query: analyze table src_orc compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_orc +POSTHOOK: Output: default@src_orc #### A masked pattern was here #### PREHOOK: query: EXPLAIN SELECT key, count(value) as cnt FROM src_orc GROUP BY key ORDER BY cnt PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out b/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out index c8190bd..4cb65be 100644 --- a/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out +++ b/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out @@ -427,40 +427,54 @@ STAGE PLANS: PREHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl +PREHOOK: Output: default@stats_tbl #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl +POSTHOOK: Output: default@stats_tbl #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2010') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2011 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2011') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2011 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2012 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt='2012') compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2012 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2012 #### A masked pattern was here #### PREHOOK: query: explain select count(*), sum(1), sum(0.2), count(1), count(s), count(bo), count(bin), count(si) from stats_tbl diff --git a/ql/src/test/results/clientpositive/llap/metadata_only_queries_with_filters.q.out b/ql/src/test/results/clientpositive/llap/metadata_only_queries_with_filters.q.out index 6dea3e0..6784f45 100644 --- a/ql/src/test/results/clientpositive/llap/metadata_only_queries_with_filters.q.out +++ b/ql/src/test/results/clientpositive/llap/metadata_only_queries_with_filters.q.out @@ -126,21 +126,29 @@ PREHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statisti PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2010 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2010 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part PREHOOK: Input: default@stats_tbl_part@dt=2014 +PREHOOK: Output: default@stats_tbl_part +PREHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns t,si,i,b,f,d,bo,s,bin POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part POSTHOOK: Input: default@stats_tbl_part@dt=2014 +POSTHOOK: Output: default@stats_tbl_part +POSTHOOK: Output: default@stats_tbl_part@dt=2014 #### A masked pattern was here #### PREHOOK: query: explain select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010 diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out index 906e5e8..d024503 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out @@ -37,12 +37,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@partitioned1 PREHOOK: Input: default@partitioned1@part=1 PREHOOK: Input: default@partitioned1@part=2 +PREHOOK: Output: default@partitioned1 +PREHOOK: Output: default@partitioned1@part=1 +PREHOOK: Output: default@partitioned1@part=2 #### A masked pattern was here #### POSTHOOK: query: analyze table partitioned1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partitioned1 POSTHOOK: Input: default@partitioned1@part=1 POSTHOOK: Input: default@partitioned1@part=2 +POSTHOOK: Output: default@partitioned1 +POSTHOOK: Output: default@partitioned1@part=1 +POSTHOOK: Output: default@partitioned1@part=2 #### A masked pattern was here #### PREHOOK: query: desc formatted partitioned1 PREHOOK: type: DESCTABLE @@ -237,12 +243,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@partitioned1 PREHOOK: Input: default@partitioned1@part=1 PREHOOK: Input: default@partitioned1@part=2 +PREHOOK: Output: default@partitioned1 +PREHOOK: Output: default@partitioned1@part=1 +PREHOOK: Output: default@partitioned1@part=2 #### A masked pattern was here #### POSTHOOK: query: analyze table partitioned1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@partitioned1 POSTHOOK: Input: default@partitioned1@part=1 POSTHOOK: Input: default@partitioned1@part=2 +POSTHOOK: Output: default@partitioned1 +POSTHOOK: Output: default@partitioned1@part=1 +POSTHOOK: Output: default@partitioned1@part=2 #### A masked pattern was here #### PREHOOK: query: desc formatted partitioned1 PREHOOK: type: DESCTABLE diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out index 5bfe159..2faf88a 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out @@ -515,7 +515,7 @@ POSTHOOK: Input: default@table_change_string_group_double insert_num c1 c2 c3 b 101 1.7976931348623157E308 1.7976931348623157E308 1.7976931348623157E308 original 102 -1.7976931348623157E308 -1.7976931348623157E308 -1.7976931348623157E308 original -103 NULL NULL NULL original +103 NULL 0.0 NULL original 104 30.774 30.774 30.774 original 105 46114.28 46114.28 46114.28 original 111 789.321 789.321 789.321 new diff --git a/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out b/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out index d01b373..f6f3aac 100644 --- a/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out +++ b/ql/src/test/results/clientpositive/llap/special_character_in_tabnames_1.q.out @@ -162,11 +162,15 @@ PREHOOK: query: analyze table `c/b/o_t1` compute statistics for columns key, val PREHOOK: type: QUERY PREHOOK: Input: default@c/b/o_t1 PREHOOK: Input: default@c/b/o_t1@dt=2014 +PREHOOK: Output: default@c/b/o_t1 +PREHOOK: Output: default@c/b/o_t1@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table `c/b/o_t1` compute statistics for columns key, value, c_int, c_float, c_boolean POSTHOOK: type: QUERY POSTHOOK: Input: default@c/b/o_t1 POSTHOOK: Input: default@c/b/o_t1@dt=2014 +POSTHOOK: Output: default@c/b/o_t1 +POSTHOOK: Output: default@c/b/o_t1@dt=2014 #### A masked pattern was here #### PREHOOK: query: analyze table `//cbo_t2` partition (dt) compute statistics PREHOOK: type: QUERY @@ -184,11 +188,15 @@ PREHOOK: query: analyze table `//cbo_t2` compute statistics for columns key, val PREHOOK: type: QUERY PREHOOK: Input: default@//cbo_t2 PREHOOK: Input: default@//cbo_t2@dt=2014 +PREHOOK: Output: default@//cbo_t2 +PREHOOK: Output: default@//cbo_t2@dt=2014 #### A masked pattern was here #### POSTHOOK: query: analyze table `//cbo_t2` compute statistics for columns key, value, c_int, c_float, c_boolean POSTHOOK: type: QUERY POSTHOOK: Input: default@//cbo_t2 POSTHOOK: Input: default@//cbo_t2@dt=2014 +POSTHOOK: Output: default@//cbo_t2 +POSTHOOK: Output: default@//cbo_t2@dt=2014 #### A masked pattern was here #### PREHOOK: query: analyze table `cbo_/t3////` compute statistics PREHOOK: type: QUERY @@ -201,10 +209,12 @@ POSTHOOK: Output: default@cbo_/t3//// PREHOOK: query: analyze table `cbo_/t3////` compute statistics for columns key, value, c_int, c_float, c_boolean PREHOOK: type: QUERY PREHOOK: Input: default@cbo_/t3//// +PREHOOK: Output: default@cbo_/t3//// #### A masked pattern was here #### POSTHOOK: query: analyze table `cbo_/t3////` compute statistics for columns key, value, c_int, c_float, c_boolean POSTHOOK: type: QUERY POSTHOOK: Input: default@cbo_/t3//// +POSTHOOK: Output: default@cbo_/t3//// #### A masked pattern was here #### PREHOOK: query: analyze table `src/_/cbo` compute statistics PREHOOK: type: QUERY @@ -217,10 +227,12 @@ POSTHOOK: Output: default@src/_/cbo PREHOOK: query: analyze table `src/_/cbo` compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@src/_/cbo +PREHOOK: Output: default@src/_/cbo #### A masked pattern was here #### POSTHOOK: query: analyze table `src/_/cbo` compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src/_/cbo +POSTHOOK: Output: default@src/_/cbo #### A masked pattern was here #### PREHOOK: query: analyze table `p/a/r/t` compute statistics PREHOOK: type: QUERY @@ -233,10 +245,12 @@ POSTHOOK: Output: default@p/a/r/t PREHOOK: query: analyze table `p/a/r/t` compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@p/a/r/t +PREHOOK: Output: default@p/a/r/t #### A masked pattern was here #### POSTHOOK: query: analyze table `p/a/r/t` compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@p/a/r/t +POSTHOOK: Output: default@p/a/r/t #### A masked pattern was here #### PREHOOK: query: analyze table `line/item` compute statistics PREHOOK: type: QUERY @@ -249,10 +263,12 @@ POSTHOOK: Output: default@line/item PREHOOK: query: analyze table `line/item` compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@line/item +PREHOOK: Output: default@line/item #### A masked pattern was here #### POSTHOOK: query: analyze table `line/item` compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@line/item +POSTHOOK: Output: default@line/item #### A masked pattern was here #### PREHOOK: query: select key, (c_int+1)+2 as x, sum(c_int) from `c/b/o_t1` group by c_float, `c/b/o_t1`.c_int, key PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/stats_only_null.q.out b/ql/src/test/results/clientpositive/llap/stats_only_null.q.out index c905ceb..57aaf55 100644 --- a/ql/src/test/results/clientpositive/llap/stats_only_null.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_only_null.q.out @@ -189,30 +189,40 @@ STAGE PLANS: PREHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null +PREHOOK: Output: default@stats_null #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null +POSTHOOK: Output: default@stats_null #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2010 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2010') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2010 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2010 #### A masked pattern was here #### PREHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=2011 +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part partition(dt='2011') compute statistics for columns a,b,c,d POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=2011 +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=2011 #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition (dt='2010') PREHOOK: type: DESCTABLE @@ -384,12 +394,18 @@ PREHOOK: type: QUERY PREHOOK: Input: default@stats_null_part PREHOOK: Input: default@stats_null_part@dt=1 PREHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +PREHOOK: Output: default@stats_null_part +PREHOOK: Output: default@stats_null_part@dt=1 +PREHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### POSTHOOK: query: analyze table stats_null_part compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_null_part POSTHOOK: Input: default@stats_null_part@dt=1 POSTHOOK: Input: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Output: default@stats_null_part +POSTHOOK: Output: default@stats_null_part@dt=1 +POSTHOOK: Output: default@stats_null_part@dt=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### PREHOOK: query: describe formatted stats_null_part partition(dt = 1) a PREHOOK: type: DESCTABLE diff --git a/ql/src/test/results/clientpositive/llap/union_remove_26.q.out b/ql/src/test/results/clientpositive/llap/union_remove_26.q.out index 18b9aa5..67fef54 100644 --- a/ql/src/test/results/clientpositive/llap/union_remove_26.q.out +++ b/ql/src/test/results/clientpositive/llap/union_remove_26.q.out @@ -103,18 +103,22 @@ POSTHOOK: Lineage: inputtbl3.val SIMPLE [(inputsrctbl3)inputsrctbl3.FieldSchema( PREHOOK: query: analyze table inputTbl1 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@inputtbl1 +PREHOOK: Output: default@inputtbl1 #### A masked pattern was here #### POSTHOOK: query: analyze table inputTbl1 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl1 +POSTHOOK: Output: default@inputtbl1 #### A masked pattern was here #### PREHOOK: query: analyze table inputTbl3 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@inputtbl3 +PREHOOK: Output: default@inputtbl3 #### A masked pattern was here #### POSTHOOK: query: analyze table inputTbl3 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl3 +POSTHOOK: Output: default@inputtbl3 #### A masked pattern was here #### PREHOOK: query: explain SELECT count(1) as rowcnt, min(val) as ms, max(val) as mx from inputTbl1 @@ -282,10 +286,12 @@ POSTHOOK: Input: default@inputtbl3 PREHOOK: query: analyze table inputTbl2 compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@inputtbl2 +PREHOOK: Output: default@inputtbl2 #### A masked pattern was here #### POSTHOOK: query: analyze table inputTbl2 compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@inputtbl2 +POSTHOOK: Output: default@inputtbl2 #### A masked pattern was here #### PREHOOK: query: explain SELECT count(1) as rowcnt, min(val) as ms, max(val) as mx from inputTbl1 diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out index 072399e..e687c5b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join1.q.out @@ -186,10 +186,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out index d79c71b..1dd24b0 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join2.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out index dbbfd34..f8d1ec2 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join3.q.out @@ -191,10 +191,12 @@ POSTHOOK: Output: default@small_alltypesorc_a PREHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_a +PREHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_a COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_a +POSTHOOK: Output: default@small_alltypesorc_a #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_a PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out index ffce9e6..a55250b 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join4.q.out @@ -201,10 +201,12 @@ POSTHOOK: Output: default@small_alltypesorc_b PREHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_alltypesorc_b +PREHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_alltypesorc_b COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_alltypesorc_b +POSTHOOK: Output: default@small_alltypesorc_b #### A masked pattern was here #### PREHOOK: query: select * from small_alltypesorc_b PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out index 4f25253..680ee42 100644 --- a/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_outer_join5.q.out @@ -27,10 +27,12 @@ POSTHOOK: Output: default@sorted_mod_4 PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@sorted_mod_4 +PREHOOK: Output: default@sorted_mod_4 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@sorted_mod_4 +POSTHOOK: Output: default@sorted_mod_4 #### A masked pattern was here #### PREHOOK: query: create table small_table stored as orc as select ctinyint, cbigint from alltypesorc limit 100 @@ -57,10 +59,12 @@ POSTHOOK: Output: default@small_table PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_table +PREHOOK: Output: default@small_table #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table +POSTHOOK: Output: default@small_table #### A masked pattern was here #### PREHOOK: query: explain vectorization detail formatted select count(*) from (select s.*, st.* @@ -267,10 +271,12 @@ POSTHOOK: Output: default@mod_8_mod_4 PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@mod_8_mod_4 +PREHOOK: Output: default@mod_8_mod_4 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@mod_8_mod_4 +POSTHOOK: Output: default@mod_8_mod_4 #### A masked pattern was here #### PREHOOK: query: create table small_table2 stored as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100 @@ -297,10 +303,12 @@ POSTHOOK: Output: default@small_table2 PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS PREHOOK: type: QUERY PREHOOK: Input: default@small_table2 +PREHOOK: Output: default@small_table2 #### A masked pattern was here #### POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS POSTHOOK: type: QUERY POSTHOOK: Input: default@small_table2 +POSTHOOK: Output: default@small_table2 #### A masked pattern was here #### PREHOOK: query: explain vectorization detail formatted select count(*) from (select s.*, st.* diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out index eb0f405..5f4735f 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out @@ -1632,10 +1632,12 @@ POSTHOOK: Output: default@dsrv_small PREHOOK: query: analyze table dsrv_small compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@dsrv_small +PREHOOK: Output: default@dsrv_small #### A masked pattern was here #### POSTHOOK: query: analyze table dsrv_small compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@dsrv_small +POSTHOOK: Output: default@dsrv_small #### A masked pattern was here #### PREHOOK: query: EXPLAIN select count(*) from dsrv_big a join dsrv_small b on (a.key_int = b.key_int) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out index b369e7c..d9fd706 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction2.q.out @@ -73,18 +73,22 @@ POSTHOOK: Output: default@dsrv2_small PREHOOK: query: analyze table dsrv2_big compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@dsrv2_big +PREHOOK: Output: default@dsrv2_big #### A masked pattern was here #### POSTHOOK: query: analyze table dsrv2_big compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@dsrv2_big +POSTHOOK: Output: default@dsrv2_big #### A masked pattern was here #### PREHOOK: query: analyze table dsrv2_small compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@dsrv2_small +PREHOOK: Output: default@dsrv2_small #### A masked pattern was here #### POSTHOOK: query: analyze table dsrv2_small compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@dsrv2_small +POSTHOOK: Output: default@dsrv2_small #### A masked pattern was here #### PREHOOK: query: EXPLAIN select count(*) from dsrv2_big a join dsrv2_small b on (a.partkey_bigint = b.partkey_bigint) PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/perf/query14.q.out b/ql/src/test/results/clientpositive/perf/query14.q.out index 9821180..051d837 100644 --- a/ql/src/test/results/clientpositive/perf/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/query14.q.out @@ -1,7 +1,7 @@ -Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 114' is a cross product -Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 115' is a cross product Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 61' is a cross product Warning: Shuffle Join MERGEJOIN[915][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 62' is a cross product +Warning: Shuffle Join MERGEJOIN[916][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 114' is a cross product +Warning: Shuffle Join MERGEJOIN[917][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 115' is a cross product Warning: Shuffle Join MERGEJOIN[912][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product Warning: Shuffle Join MERGEJOIN[913][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product PREHOOK: query: explain diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 20c330a..32609eb 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -234,10 +234,12 @@ Stage-2 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@src_stats +PREHOOK: Output: default@src_stats #### A masked pattern was here #### POSTHOOK: query: analyze table src_stats compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stats +POSTHOOK: Output: default@src_stats #### A masked pattern was here #### PREHOOK: query: explain analyze analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -246,19 +248,21 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-2 +Stage-3 Column Stats Work{} - Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=10) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Stage-2 + Stats-Aggr Operator + Stage-0 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=960) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500/500 width=10) + Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=10) + default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] PREHOOK: query: CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) PREHOOK: type: CREATEMACRO diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out index ee9affb..b35e294 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out @@ -36,10 +36,12 @@ Stage-2 PREHOOK: query: analyze table src_stats compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@src_stats +PREHOOK: Output: default@src_stats #### A masked pattern was here #### POSTHOOK: query: analyze table src_stats compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@src_stats +POSTHOOK: Output: default@src_stats #### A masked pattern was here #### PREHOOK: query: explain analyze analyze table src_stats compute statistics for columns PREHOOK: type: QUERY @@ -48,19 +50,21 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-2 +Stage-3 Column Stats Work{} - Stage-0 - Reducer 2 - File Output Operator [FS_5] - Group By Operator [GBY_3] (rows=1/1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=500/500 width=10) - Output:["key","value"] - TableScan [TS_0] (rows=500/500 width=10) - default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] + Stage-2 + Stats-Aggr Operator + Stage-0 + Reducer 2 + File Output Operator [FS_5] + Group By Operator [GBY_3] (rows=1/1 width=960) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 16)","compute_stats(VALUE._col2, 16)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_2] + Select Operator [SEL_1] (rows=500/500 width=10) + Output:["key","value"] + TableScan [TS_0] (rows=500/500 width=10) + default@src_stats,src_stats,Tbl:COMPLETE,Col:NONE,Output:["key","value"] PREHOOK: query: drop table src_multi2 PREHOOK: type: DROPTABLE diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 74e4693..da52b0a 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -19,10 +19,12 @@ POSTHOOK: Lineage: acid_vectorized.b SIMPLE [(alltypesorc)alltypesorc.FieldSchem PREHOOK: query: analyze table acid_vectorized compute statistics for columns PREHOOK: type: QUERY PREHOOK: Input: default@acid_vectorized +PREHOOK: Output: default@acid_vectorized #### A masked pattern was here #### POSTHOOK: query: analyze table acid_vectorized compute statistics for columns POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_vectorized +POSTHOOK: Output: default@acid_vectorized #### A masked pattern was here #### PREHOOK: query: explain select a, b from acid_vectorized order by a, b PREHOOK: type: QUERY @@ -39,13 +41,13 @@ Stage-0 Stage-1 Reducer 2 vectorized File Output Operator [FS_8] - Select Operator [SEL_7] (rows=16 width=101) + Select Operator [SEL_7] (rows=10 width=101) Output:["_col0","_col1"] <-Map 1 [SIMPLE_EDGE] vectorized SHUFFLE [RS_6] - Select Operator [SEL_5] (rows=16 width=101) + Select Operator [SEL_5] (rows=10 width=101) Output:["_col0","_col1"] - TableScan [TS_0] (rows=16 width=101) + TableScan [TS_0] (rows=10 width=101) default@acid_vectorized,acid_vectorized, ACID table,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b"] PREHOOK: query: explain select key, value @@ -200,21 +202,23 @@ POSTHOOK: type: QUERY Vertex dependency in root stage Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -Stage-2 +Stage-3 Column Stats Work{} - Stage-0 - Reducer 2 - File Output Operator [FS_6] - Group By Operator [GBY_4] (rows=1 width=960) - Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] - PARTITION_ONLY_SHUFFLE [RS_3] - Group By Operator [GBY_2] (rows=1 width=984) - Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"] - Select Operator [SEL_1] (rows=500 width=178) - Output:["key","value"] - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + Stage-2 + Stats-Aggr Operator + Stage-0 + Reducer 2 + File Output Operator [FS_6] + Group By Operator [GBY_4] (rows=1 width=960) + Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"] + <-Map 1 [CUSTOM_SIMPLE_EDGE] + PARTITION_ONLY_SHUFFLE [RS_3] + Group By Operator [GBY_2] (rows=1 width=984) + Output:["_col0","_col1"],aggregations:["compute_stats(key, 16)","compute_stats(value, 16)"] + Select Operator [SEL_1] (rows=500 width=178) + Output:["key","value"] + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: explain CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x))