diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 3ec63ee..6fe0814 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -941,39 +941,7 @@ protected FSPaths getDynOutPaths(List row, String lbDirName) throws Hive + "Maximum was set to: " + maxPartitions); } - if (!conf.getDpSortState().equals(DPSortState.NONE) && prevFsp != null) { - // close the previous fsp as it is no longer needed - prevFsp.closeWriters(false); - - // since we are closing the previous fsp's record writers, we need to see if we can get - // stats from the record writer and store in the previous fsp that is cached - if (conf.isGatherStats() && isCollectRWStats) { - SerDeStats stats = null; - if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { - RecordWriter outWriter = prevFsp.outWriters[0]; - if (outWriter != null) { - stats = ((StatsProvidingRecordWriter) outWriter).getStats(); - } - } else if (prevFsp.updaters[0] != null) { - stats = prevFsp.updaters[0].getStats(); - } - if (stats != null) { - prevFsp.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize()); - prevFsp.stat.addToStat(StatsSetupConst.ROW_COUNT, stats.getRowCount()); - } - } - - // let writers release the memory for garbage collection - prevFsp.outWriters[0] = null; - - prevFsp = null; - } - fsp2 = createNewPaths(dpDir); - if (prevFsp == null) { - prevFsp = fsp2; - } - if(conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) { createBucketForFileIdx(fsp2, 0); valToPaths.put(pathKey, fsp2); diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid_stats_optimizer.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid_stats_optimizer.q new file mode 100644 index 0000000..367132e --- /dev/null +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid_stats_optimizer.q @@ -0,0 +1,119 @@ +set hive.mapred.mode=nonstrict; +set hive.compute.query.using.stats=true; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.optimize.sort.dynamic.partition=false; + +-- single level partition, sorted dynamic partition disabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds) select key,value,ds from srcpart; +select count(*) from acid where ds='2008-04-08'; + +insert into table acid partition(ds='2008-04-08') values("foo", "bar"); +select count(*) from acid where ds='2008-04-08'; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08'; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08'); +update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08'); +select count(*) from acid where ds in ('2008-04-08'); + +delete from acid where key = 'foo' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +set hive.optimize.sort.dynamic.partition=true; + +-- single level partition, sorted dynamic partition enabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds) select key,value,ds from srcpart; +select count(*) from acid where ds='2008-04-08'; + +insert into table acid partition(ds='2008-04-08') values("foo", "bar"); +select count(*) from acid where ds='2008-04-08'; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08'; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08'); +update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08'); +select count(*) from acid where ds in ('2008-04-08'); + +delete from acid where key = 'foo' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +set hive.optimize.sort.dynamic.partition=false; + +-- 2 level partition, sorted dynamic partition disabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds,hr) select * from srcpart; +select count(*) from acid where ds='2008-04-08' and hr=11; + +insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar"); +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11; +select count(*) from acid where ds='2008-04-08' and hr>=11; + +delete from acid where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +set hive.optimize.sort.dynamic.partition=true; + +-- 2 level partition, sorted dynamic partition enabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds,hr) select * from srcpart; +select count(*) from acid where ds='2008-04-08' and hr=11; + +insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar"); +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11; +select count(*) from acid where ds='2008-04-08' and hr>=11; + +delete from acid where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.constant.propagation=false; + +-- 2 level partition, sorted dynamic partition enabled, constant propagation disabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds,hr) select * from srcpart; +select count(*) from acid where ds='2008-04-08' and hr=11; + +insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar"); +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11; +update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11; +select count(*) from acid where ds='2008-04-08' and hr>=11; + +delete from acid where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +set hive.optimize.sort.dynamic.partition=true; diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid_stats_optimizer.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid_stats_optimizer.q.out new file mode 100644 index 0000000..2dec1a2 --- /dev/null +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid_stats_optimizer.q.out @@ -0,0 +1,1246 @@ +PREHOOK: query: -- single level partition, sorted dynamic partition disabled +drop table acid +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- single level partition, sorted dynamic partition disabled +drop table acid +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-09 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1000 +PREHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), ds (type: string) + outputColumnNames: _col0, _col3 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1000 +PREHOOK: query: -- single level partition, sorted dynamic partition enabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- single level partition, sorted dynamic partition enabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-09 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1000 +PREHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), ds (type: string) + outputColumnNames: _col0, _col3 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + value expressions: _col3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col2 (type: string) + outputColumnNames: _col0, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: string), '_bucket_number' (type: string), _col0 (type: struct) + sort order: +++ + Map-reduce partition columns: _col3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY._col0 (type: struct), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string), KEY.'_bucket_number' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, '_bucket_number' + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1000 +PREHOOK: query: -- 2 level partition, sorted dynamic partition disabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- 2 level partition, sorted dynamic partition disabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__3 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__3 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), hr (type: int) + outputColumnNames: _col0, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +500 +PREHOOK: query: -- 2 level partition, sorted dynamic partition enabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- 2 level partition, sorted dynamic partition enabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__4 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__4 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct) + outputColumnNames: _col0 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), hr (type: int) + outputColumnNames: _col0, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + value expressions: _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col3 (type: int) + outputColumnNames: _col0, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: '2008-04-08' (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct) + sort order: ++++ + Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY._col0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number' + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +500 +PREHOOK: query: -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__5 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__5 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), key (type: string), 'bar' (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct) + sort order: ++++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: int) + value expressions: _col1 (type: string), _col2 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number' + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (key = 'foo') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), key (type: string), 'bar' (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct) + sort order: ++++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: int) + value expressions: _col1 (type: string), _col2 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY.'_bucket_number' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, '_bucket_number' + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: query: update acid set value = 'bar' where key = 'foo' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +#### A masked pattern was here #### +500