diff --git a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java index 41d150c..1466b69 100644 --- a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@ -222,16 +222,6 @@ public static void setBasicStatsState(Map params, String setting // old format of statsAcc, e.g., TRUE or FALSE LOG.debug("In StatsSetupConst, JsonParser can not parse statsAcc."); stats = new JSONObject(new LinkedHashMap()); - try { - if (statsAcc.equals(TRUE)) { - stats.put(BASIC_STATS, TRUE); - } else { - stats.put(BASIC_STATS, FALSE); - } - } catch (JSONException e1) { - // impossible to throw any json exceptions. - LOG.trace(e1.getMessage()); - } } if (!stats.has(BASIC_STATS)) { // duplicate key is not possible @@ -332,4 +322,13 @@ public static void clearColumnStatsState(Map params) { params.put(COLUMN_STATS_ACCURATE, stats.toString()); } } + + public static void setBasicStatsStateForCreateTable(Map params, String setting) { + if (TRUE.equals(setting)) { + for (String stat : StatsSetupConst.supportedStats) { + params.put(stat, "0"); + } + } + setBasicStatsState(params, setting); + } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index c046708..76c1636 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -103,7 +103,7 @@ protected void setUp() { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, false, false); + db.loadTable(hadoopDataFile[i], src, false, false, false, false, false); i++; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 866e1c3..c55f398 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -265,6 +265,13 @@ public static void populateQuickStats(FileStatus[] fileStatus, Map parameters = tTable.getParameters(); + // In the following scenarios, we need to reset the stats to true. + // work.getTableSpecs() != null means analyze command + // work.getLoadTableDesc().getReplace() is true means insert overwrite command etc. + if (work.getTableSpecs() != null + || work.getLoadTableDesc().getReplace()) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); + } // non-partitioned tables: if (!existStats(parameters) && atomic) { return 0; @@ -182,7 +189,7 @@ private int aggregateStats(Hive db) { updateQuickStats(wh, parameters, tTable.getSd()); // write table stats to metastore - if (!getWork().getNoStatsAggregator()) { + if (!getWork().getNoStatsAggregator() && StatsSetupConst.areBasicStatsUptoDate(parameters)) { environmentContext = new EnvironmentContext(); environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); } @@ -203,6 +210,10 @@ private int aggregateStats(Hive db) { // org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); Map parameters = tPart.getParameters(); + if (work.getTableSpecs() != null + || work.getLoadTableDesc().getReplace()) { + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); + } if (!existStats(parameters) && atomic) { continue; } @@ -220,7 +231,8 @@ private int aggregateStats(Hive db) { updateQuickStats(wh, parameters, tPart.getSd()); - if (!getWork().getNoStatsAggregator()) { + if (!getWork().getNoStatsAggregator() + && StatsSetupConst.areBasicStatsUptoDate(parameters)) { environmentContext = new EnvironmentContext(); environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK); @@ -352,6 +364,7 @@ private void clearStats(Map parameters) { parameters.remove(statType); } } + StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); } private String toString(Map parameters) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index c27481f..bfdddf5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1753,10 +1753,12 @@ private void constructOneLBLocationMap(FileStatus fSta, * If the source directory is LOCAL * @param isSkewedStoreAsSubdir * if list bucketing enabled + * @param hasFollowingStatsTask + * if there is any following stats task * @param isAcid true if this is an ACID based write */ - public void loadTable(Path loadPath, String tableName, boolean replace, - boolean isSrcLocal, boolean isSkewedStoreAsSubdir, boolean isAcid) + public void loadTable(Path loadPath, String tableName, boolean replace, boolean isSrcLocal, + boolean isSkewedStoreAsSubdir, boolean isAcid, boolean hasFollowingStatsTask) throws HiveException { List newFiles = null; @@ -1798,8 +1800,13 @@ public void loadTable(Path loadPath, String tableName, boolean replace, throw new HiveException(e); } + EnvironmentContext environmentContext = null; + if (hasFollowingStatsTask) { + environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + } try { - alterTable(tableName, tbl, null); + alterTable(tableName, tbl, environmentContext); } catch (InvalidOperationException e) { throw new HiveException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 549d24f..bba6463 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -286,7 +286,7 @@ private AddPartitionDesc getBaseAddPartitionDescFromPartition( private CreateTableDesc getBaseCreateTableDescFromTable(String dbName, org.apache.hadoop.hive.metastore.api.Table table) { if ((table.getPartitionKeys() == null) || (table.getPartitionKeys().size() == 0)){ - table.putToParameters(StatsSetupConst.DO_NOT_UPDATE_STATS,"true"); + table.putToParameters(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); } CreateTableDesc tblDesc = new CreateTableDesc( dbName, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java index 03b4d8b..3da5208 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -777,7 +778,15 @@ public Table toTable(HiveConf conf) throws HiveException { } } } + if (getLocation() == null && !this.isCTAS) { + StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(), + StatsSetupConst.TRUE); + } else { + StatsSetupConst.setBasicStatsStateForCreateTable(tbl.getTTable().getParameters(), + StatsSetupConst.FALSE); + } return tbl; } + } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index eaeb66b..95d00f2 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -141,7 +141,7 @@ db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, true, false, false); + db.loadTable(hadoopDataFile[i], src, false, true, false, false, false); i++; } diff --git a/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q b/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q new file mode 100644 index 0000000..73f5243 --- /dev/null +++ b/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q @@ -0,0 +1,121 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.compute.query.using.stats=true; + +create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc TBLPROPERTIES ('transactional'='true'); + +desc formatted acid_ivot; + +LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot; + +desc formatted acid_ivot; + +explain select count(*) from acid_ivot; + +select count(*) from acid_ivot; + +insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true); + +desc formatted acid_ivot; + +explain select count(*) from acid_ivot; + +select count(*) from acid_ivot; + +drop table acid_ivot; + +create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc TBLPROPERTIES ('transactional'='true'); + +insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true); + +desc formatted acid_ivot; + +explain select count(*) from acid_ivot; + +select count(*) from acid_ivot; + +insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true); + +desc formatted acid_ivot; + +explain select count(*) from acid_ivot; + +select count(*) from acid_ivot; + +LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot; + +desc formatted acid_ivot; + +explain select count(*) from acid_ivot; + +drop table acid_ivot; + +create table acid_ivot like src; + +desc formatted acid_ivot; + +insert overwrite table acid_ivot select * from src; + +desc formatted acid_ivot; + +explain select count(*) from acid_ivot; + +select count(*) from acid_ivot; + +CREATE TABLE sp (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE sp PARTITION (ds="2008-04-08", hr="11"); + +desc formatted sp PARTITION (ds="2008-04-08", hr="11"); + +explain select count(*) from sp where ds="2008-04-08" and hr="11"; + +select count(*) from sp where ds="2008-04-08" and hr="11"; + +insert into table sp PARTITION (ds="2008-04-08", hr="11") values + ('1', '2'), ('3', '4'); + +desc formatted sp PARTITION (ds="2008-04-08", hr="11"); + +analyze table sp PARTITION (ds="2008-04-08", hr="11") compute statistics; + +desc formatted sp PARTITION (ds="2008-04-08", hr="11"); + +explain select count(*) from sp where ds="2008-04-08" and hr="11"; + +select count(*) from sp where ds="2008-04-08" and hr="11"; + diff --git a/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out new file mode 100644 index 0000000..094b9eb --- /dev/null +++ b/ql/src/test/results/clientpositive/insert_values_orig_table_use_metadata.q.out @@ -0,0 +1,984 @@ +PREHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + transactional true +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [cint] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"false\"} + numFiles 1 + totalSize 377237 + transactional true +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [cint] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from acid_ivot +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from acid_ivot +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid_ivot + Statistics: Num rows: 1 Data size: 377237 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 377237 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +12288 +PREHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid_ivot +POSTHOOK: Lineage: acid_ivot.cbigint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cdouble EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cfloat EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.csmallint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring2 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctinyint EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"false\"} + numFiles 2 + totalSize 378741 + transactional true +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [cint] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from acid_ivot +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from acid_ivot +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid_ivot + Statistics: Num rows: 1 Data size: 378741 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 378741 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +12290 +PREHOOK: query: drop table acid_ivot +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid_ivot +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: drop table acid_ivot +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid_ivot +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: create table acid_ivot( + ctinyint TINYINT, + csmallint SMALLINT, + cint INT, + cbigint BIGINT, + cfloat FLOAT, + cdouble DOUBLE, + cstring1 STRING, + cstring2 STRING, + ctimestamp1 TIMESTAMP, + ctimestamp2 TIMESTAMP, + cboolean1 BOOLEAN, + cboolean2 BOOLEAN) clustered by (cint) into 1 buckets stored as orc TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@acid_ivot +POSTHOOK: Lineage: acid_ivot.cbigint EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean2 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cdouble EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cfloat EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cint EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.csmallint EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp1 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp2 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctinyint EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 2 + rawDataSize 0 + totalSize 1508 + transactional true +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [cint] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from acid_ivot +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from acid_ivot +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +2 +PREHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__3 +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: insert into table acid_ivot values + (1, 2, 3, 4, 3.14, 2.34, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true), + (111, 222, 3333, 444, 13.14, 10239302.34239320, 'fred', 'bob', '2014-09-01 10:34:23.111', '1944-06-06 06:00:00', true, true) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__3 +POSTHOOK: Output: default@acid_ivot +POSTHOOK: Lineage: acid_ivot.cbigint EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean1 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col11, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cboolean2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col12, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cdouble EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col6, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cfloat EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col5, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cint EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.csmallint EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring1 SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col7, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.cstring2 SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col8, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp1 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col9, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctimestamp2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col10, type:string, comment:), ] +POSTHOOK: Lineage: acid_ivot.ctinyint EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 4 + rawDataSize 0 + totalSize 3016 + transactional true +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [cint] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from acid_ivot +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from acid_ivot +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +4 +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/alltypesorc" into table acid_ivot +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +ctinyint tinyint +csmallint smallint +cint int +cbigint bigint +cfloat float +cdouble double +cstring1 string +cstring2 string +ctimestamp1 timestamp +ctimestamp2 timestamp +cboolean1 boolean +cboolean2 boolean + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"false\"} + numFiles 3 + totalSize 380253 + transactional true +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde +InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [cint] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from acid_ivot +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from acid_ivot +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid_ivot + Statistics: Num rows: 1 Data size: 380253 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 380253 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table acid_ivot +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid_ivot +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: drop table acid_ivot +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid_ivot +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: create table acid_ivot like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: create table acid_ivot like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_ivot +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: insert overwrite table acid_ivot select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@acid_ivot +POSTHOOK: query: insert overwrite table acid_ivot select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@acid_ivot +POSTHOOK: Lineage: acid_ivot.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid_ivot.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc formatted acid_ivot +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@acid_ivot +POSTHOOK: query: desc formatted acid_ivot +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@acid_ivot +# col_name data_type comment + +key string default +value string default + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 500 + rawDataSize 5312 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from acid_ivot +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from acid_ivot +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from acid_ivot +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid_ivot +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_ivot +#### A masked pattern was here #### +500 +PREHOOK: query: CREATE TABLE sp (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sp +POSTHOOK: query: CREATE TABLE sp (key STRING COMMENT 'default', value STRING COMMENT 'default') +PARTITIONED BY (ds STRING, hr STRING) +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sp +PREHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE sp PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@sp +POSTHOOK: query: LOAD DATA LOCAL INPATH "../../data/files/kv1.txt" +OVERWRITE INTO TABLE sp PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@sp +POSTHOOK: Output: default@sp@ds=2008-04-08/hr=11 +PREHOOK: query: desc formatted sp PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sp +POSTHOOK: query: desc formatted sp PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sp +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: sp +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"false\"} + numFiles 1 + totalSize 5812 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from sp where ds="2008-04-08" and hr="11" +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from sp where ds="2008-04-08" and hr="11" +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: sp + Statistics: Num rows: 1 Data size: 5812 Basic stats: PARTIAL Column stats: NONE + Select Operator + Statistics: Num rows: 1 Data size: 5812 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from sp where ds="2008-04-08" and hr="11" +PREHOOK: type: QUERY +PREHOOK: Input: default@sp +PREHOOK: Input: default@sp@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from sp where ds="2008-04-08" and hr="11" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sp +POSTHOOK: Input: default@sp@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table sp PARTITION (ds="2008-04-08", hr="11") values + ('1', '2'), ('3', '4') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__4 +PREHOOK: Output: default@sp@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table sp PARTITION (ds="2008-04-08", hr="11") values + ('1', '2'), ('3', '4') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__4 +POSTHOOK: Output: default@sp@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: sp PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: sp PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: desc formatted sp PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sp +POSTHOOK: query: desc formatted sp PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sp +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: sp +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"false\"} + numFiles 2 + totalSize 5820 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: analyze table sp PARTITION (ds="2008-04-08", hr="11") compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@sp +PREHOOK: Input: default@sp@ds=2008-04-08/hr=11 +PREHOOK: Output: default@sp +PREHOOK: Output: default@sp@ds=2008-04-08/hr=11 +POSTHOOK: query: analyze table sp PARTITION (ds="2008-04-08", hr="11") compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sp +POSTHOOK: Input: default@sp@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@sp +POSTHOOK: Output: default@sp@ds=2008-04-08/hr=11 +PREHOOK: query: desc formatted sp PARTITION (ds="2008-04-08", hr="11") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@sp +POSTHOOK: query: desc formatted sp PARTITION (ds="2008-04-08", hr="11") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@sp +# col_name data_type comment + +key string default +value string default + +# Partition Information +# col_name data_type comment + +ds string +hr string + +# Detailed Partition Information +Partition Value: [2008-04-08, 11] +Database: default +Table: sp +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 502 + rawDataSize 5318 + totalSize 5820 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: explain select count(*) from sp where ds="2008-04-08" and hr="11" +PREHOOK: type: QUERY +POSTHOOK: query: explain select count(*) from sp where ds="2008-04-08" and hr="11" +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from sp where ds="2008-04-08" and hr="11" +PREHOOK: type: QUERY +PREHOOK: Input: default@sp +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from sp where ds="2008-04-08" and hr="11" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sp +#### A masked pattern was here #### +502