diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index d79879c..c714579 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -18,14 +18,8 @@ package org.apache.hadoop.hive.ql.optimizer; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Stack; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -72,8 +66,14 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; /** * When dynamic partitioning (with or without bucketing and sorting) is enabled, this optimization @@ -157,7 +157,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // the reduce sink key. Since both key columns are not prefix subset // ReduceSinkDeDuplication will not merge them together resulting in 2 MR jobs. // To avoid that we will remove the RS (and EX) inserted by enforce bucketing/sorting. - removeRSInsertedByEnforceBucketing(fsOp); + if (!removeRSInsertedByEnforceBucketing(fsOp)) { + LOG.debug("Bailing out of sort dynamic partition optimization as some partition columns " + + "got constant folded."); + return null; + } // unlink connection between FS and its parent Operator fsParent = fsOp.getParentOperators().get(0); @@ -263,7 +267,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // Remove RS and EX introduced by enforce bucketing/sorting config // Convert PARENT -> RS -> EX -> FS to PARENT -> FS - private void removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { + private boolean removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { HiveConf hconf = parseCtx.getConf(); boolean enforceBucketing = HiveConf.getBoolVar(hconf, ConfVars.HIVEENFORCEBUCKETING); boolean enforceSorting = HiveConf.getBoolVar(hconf, ConfVars.HIVEENFORCESORTING); @@ -298,17 +302,27 @@ private void removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { Operator rsGrandChild = rsChild.getChildOperators().get(0); if (rsChild instanceof ExtractOperator) { + // if schema size cannot be matched, then it could be because of constant folding + // converting partition column expression to constant expression. The constant + // expression will then get pruned by column pruner since it will not reference to + // any columns. + if (rsParent.getSchema().getSignature().size() != + rsChild.getSchema().getSignature().size()) { + return false; + } rsParent.getChildOperators().clear(); rsParent.getChildOperators().add(rsGrandChild); rsGrandChild.getParentOperators().clear(); rsGrandChild.getParentOperators().add(rsParent); parseCtx.removeOpParseCtx(rsToRemove); parseCtx.removeOpParseCtx(rsChild); - LOG.info("Removed " + rsParent.getOperatorId() + " and " + rsChild.getOperatorId() + LOG.info("Removed " + rsToRemove.getOperatorId() + " and " + rsChild.getOperatorId() + " as it was introduced by enforce bucketing/sorting."); } } } + + return true; } private List getPartitionPositions(DynamicPartitionCtx dpCtx, RowSchema schema) { diff --git ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q new file mode 100644 index 0000000..f2c0f0b --- /dev/null +++ ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q @@ -0,0 +1,117 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +set hive.optimize.sort.dynamic.partition=false; + +-- single level partition, sorted dynamic partition disabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds) select key,value,ds from srcpart; +select count(*) from acid where ds='2008-04-08'; + +insert into table acid partition(ds='2008-04-08') values("foo", "bar"); +select count(*) from acid where ds='2008-04-08'; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08'; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +explain update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08'); +update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08'); +select count(*) from acid where ds in ('2008-04-08'); + +delete from acid where key = 'foo' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +set hive.optimize.sort.dynamic.partition=true; + +-- single level partition, sorted dynamic partition enabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds) select key,value,ds from srcpart; +select count(*) from acid where ds='2008-04-08'; + +insert into table acid partition(ds='2008-04-08') values("foo", "bar"); +select count(*) from acid where ds='2008-04-08'; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08'; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +explain update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08'); +update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08'); +select count(*) from acid where ds in ('2008-04-08'); + +delete from acid where key = 'foo' and ds='2008-04-08'; +select count(*) from acid where ds='2008-04-08'; + +set hive.optimize.sort.dynamic.partition=false; + +-- 2 level partition, sorted dynamic partition disabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds,hr) select * from srcpart; +select count(*) from acid where ds='2008-04-08' and hr=11; + +insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar"); +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11; +select count(*) from acid where ds='2008-04-08' and hr>=11; + +delete from acid where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +set hive.optimize.sort.dynamic.partition=true; + +-- 2 level partition, sorted dynamic partition enabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds,hr) select * from srcpart; +select count(*) from acid where ds='2008-04-08' and hr=11; + +insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar"); +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11; +select count(*) from acid where ds='2008-04-08' and hr>=11; + +delete from acid where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.constant.propagation=false; + +-- 2 level partition, sorted dynamic partition enabled, constant propagation disabled +drop table acid; +CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid partition(ds,hr) select * from srcpart; +select count(*) from acid where ds='2008-04-08' and hr=11; + +insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar"); +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11; +update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11; +select count(*) from acid where ds='2008-04-08' and hr>=11; + +delete from acid where key = 'foo' and ds='2008-04-08' and hr=11; +select count(*) from acid where ds='2008-04-08' and hr=11; + +set hive.optimize.sort.dynamic.partition=true; diff --git ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out new file mode 100644 index 0000000..8ebbdc3 --- /dev/null +++ ql/src/test/results/clientpositive/dynpart_sort_optimization_acid.q.out @@ -0,0 +1,1264 @@ +PREHOOK: query: -- single level partition, sorted dynamic partition disabled +drop table acid +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- single level partition, sorted dynamic partition disabled +drop table acid +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-09 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1000 +PREHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), '2008-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col3 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string), _col3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1000 +PREHOOK: query: -- single level partition, sorted dynamic partition enabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- single level partition, sorted dynamic partition enabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds) select key,value,ds from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-09 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1000 +PREHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08') values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).key SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08).value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), '2008-04-08' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1001 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string), ds (type: string) + outputColumnNames: _col0, _col1, _col3 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string), _col3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds in ('2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds in ('2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +PREHOOK: Output: default@acid@ds=2008-04-08 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +POSTHOOK: Output: default@acid@ds=2008-04-08 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08 +#### A masked pattern was here #### +1000 +PREHOOK: query: -- 2 level partition, sorted dynamic partition disabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- 2 level partition, sorted dynamic partition disabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__3 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__3 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), '2008-04-08' (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500 +PREHOOK: query: -- 2 level partition, sorted dynamic partition enabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- 2 level partition, sorted dynamic partition enabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__4 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__4 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string) + outputColumnNames: _col0, _col1 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) + value expressions: _col1 (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), '2008-04-08' (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-2 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500 +PREHOOK: query: -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled +drop table acid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@acid +PREHOOK: Output: default@acid +POSTHOOK: query: -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled +drop table acid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@acid +POSTHOOK: Output: default@acid +PREHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid +POSTHOOK: query: CREATE TABLE acid(key string, value string) PARTITIONED BY(ds string, hr int) CLUSTERED BY(key) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid +PREHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@acid +POSTHOOK: query: insert into table acid partition(ds,hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500 +PREHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__5 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into table acid partition(ds='2008-04-08',hr=11) values("foo", "bar") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__5 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(values__tmp__table__5)values__tmp__table__5.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct) + sort order: ++++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: int) + value expressions: _col0 (type: struct), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string) + Reduce Operator Tree: + Extract + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +501 +PREHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +POSTHOOK: query: explain update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: acid + Filter Operator + predicate: (value = 'bar') (type: boolean) + Select Operator + expressions: ROW__ID (type: struct), 'foo' (type: string), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Reduce Output Operator + key expressions: _col0 (type: struct) + sort order: + + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string), _col0 (type: struct) + sort order: ++++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: int) + value expressions: _col0 (type: struct), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), '_bucket_number' (type: string) + Reduce Operator Tree: + Extract + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.acid + + Stage: Stage-3 + Stats-Aggr Operator + +PREHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: query: update acid set key = 'foo' where value = 'bar' and ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=12 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Input: default@acid@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr>=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=12 +#### A masked pattern was here #### +1001 +PREHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +PREHOOK: Output: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: query: delete from acid where key = 'foo' and ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@acid@ds=2008-04-08/hr=11 +PREHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid +PREHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from acid where ds='2008-04-08' and hr=11 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid +POSTHOOK: Input: default@acid@ds=2008-04-08/hr=11 +#### A masked pattern was here #### +500