diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index 80dbbee5bad6d99144be9cfac8368db9cf9a7b36..a1746fcad8ae78a5c73be84f032607a1e6ce00c9 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -84,8 +84,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -108,12 +107,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - null sort order: - sort order: - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col1 (type: string), _bucket_number (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col1 (type: string) tag: -1 - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -159,13 +158,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _bucket_number File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### + Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### @@ -194,34 +193,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: id, key - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(id, 'hll') - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Move Operator @@ -263,83 +234,6 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: struct) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10002 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index a8cdc8fe057b8725a4d676dee434723fc8899b0a..ba3752d6a6e3383e4ac77bdc61363f032e39d0b4 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -102,8 +102,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -126,12 +125,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - null sort order: - sort order: - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col1 (type: string), _bucket_number (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col1 (type: string) tag: -1 - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -177,13 +176,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _bucket_number File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### + Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### @@ -212,34 +211,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: id, key - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(id, 'hll') - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Move Operator @@ -281,83 +252,6 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: struct) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10002 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out index 826fae9d422d3423f8ab32997348ce246af11938..7b2561358a95285fbfaaa6d196e35a0bea62ce85 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +200 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +50 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +100 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +300 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +350 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -50 +400 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -50 +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out index bb63070503051415b5e947f02dfe4c3660feb955..1201ce2107cdd428eb481f48ea5444a3ac71d68b 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +200 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -100 +300 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -200 +400 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -300 +500 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out index 7758dc9d53a7872818ed16097c7524be3e11f83e..0931e3d5576c87ee918e1a86502e2a208e976513 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +200 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +50 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +100 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +300 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +350 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -50 +400 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -50 +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out index 0ccd0e43452c7f1826524d61f29792f165560edd..15ae3d91a75ca8cbe3d7e036ac2ff7eef87ec104 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +200 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -100 +300 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -200 +400 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -300 +500 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out index 340791aa7ffeff777c28be2335b2733263afa1da..24fc5258043465cc6a00a271ca3e797884063aaf 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +200 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +50 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +100 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +300 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +350 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -50 +400 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -50 +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out index 160842236b55c2cef4ed211708ac89e167bb48f4..6bcfe41262dee389e854aa399dcc854a2b16599b 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +200 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -100 +300 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -200 +400 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fb50588fe7ce06df6bf41775dbef42143d3d8bb9..141d09351e5a01cf8a49952b37da65ba75dba2d5 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1138,6 +1138,7 @@ spark.query.files=add_part_multiple.q, \ decimal_1_1.q, \ decimal_join.q, \ disable_merge_for_bucketing.q, \ + dynpart_sort_optimization.q, \ enforce_order.q, \ escape_clusterby1.q, \ escape_distributeby1.q, \ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index ff35e8c082088fc93bed996d83d64e9892f20f2a..d02633458273a084c166539a9663ab48001ae33e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; +import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.orc.OrcConf; @@ -706,7 +707,11 @@ private boolean shouldDo(List partitionPos, Operator colStats = fsParent.getStatistics().getColumnStats(); + Statistics tStats = fsParent.getStatistics(); + if ( tStats == null ) + return true; + + List colStats = tStats.getColumnStats(); if (colStats == null || colStats.isEmpty()) { return true; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java index 302b3506f361fe72283257b7ec7d8740938c4a0c..76ebcf8e94f7249afb5c79f5b804f389116e76a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java @@ -118,6 +118,12 @@ protected void setInputFormat(Task task) { } } + @Override + protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, + Set outputs) throws SemanticException { + this.runDynPatitionSortOptimizations(pCtx, conf); + } + private void setInputFormat(MapWork work, Operator op) { if (op.isUseBucketizedHiveInputFormat()) { work.setUseBucketizedHiveInputFormat(true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index baefe1b39eb1936f8a5dd1e2be5bbf3bfb73d63b..c6bd8b1b7f5f0c4a31c339b1781a8c47fc63b8d6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -48,8 +48,12 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; +import org.apache.hadoop.hive.ql.optimizer.NonBlockingOpDeDupProc; +import org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; +import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext; import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; @@ -654,6 +658,30 @@ protected abstract void optimizeTaskPlan(List> root protected abstract void generateTaskTree(List> rootTasks, ParseContext pCtx, List> mvTask, Set inputs, Set outputs) throws SemanticException; + /* + * Called for dynamic partitioning sort optimization. + */ + protected void runDynPatitionSortOptimizations(ParseContext parseContext, HiveConf hConf) throws SemanticException { + // run Sorted dynamic partition optimization + + if(HiveConf.getBoolVar(hConf, HiveConf.ConfVars.DYNAMICPARTITIONING) && + HiveConf.getVar(hConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") && + !HiveConf.getBoolVar(hConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { + new SortedDynPartitionOptimizer().transform(parseContext); + + if(HiveConf.getBoolVar(hConf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION) + || parseContext.hasAcidWrite()) { + + // Dynamic sort partition adds an extra RS therefore need to de-dup + new ReduceSinkDeDuplication().transform(parseContext); + // there is an issue with dedup logic wherein SELECT is created with wrong columns + // NonBlockingOpDeDupProc fixes that + new NonBlockingOpDeDupProc().transform(parseContext); + + } + } + } + /** * Create a clone of the parse context */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 0a76ffa28bd9476d2a14083d2ef4522fa46c3868..851c4bd3e22483aeb5e5c9c1d00eb9b782274f9e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -64,7 +64,10 @@ import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx; import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization; +import org.apache.hadoop.hive.ql.optimizer.NonBlockingOpDeDupProc; +import org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer; import org.apache.hadoop.hive.ql.optimizer.SparkRemoveDynamicPruning; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer; @@ -120,6 +123,9 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // Annotation OP tree with statistics runStatsAnnotation(procCtx); + // Run Dynamic Partitioning sort Optimization. + runDynPatitionSortOptimizations(procCtx); + // Set reducer parallelism runSetReducerParallelism(procCtx); @@ -336,6 +342,13 @@ private void runJoinOptimizations(OptimizeSparkProcContext procCtx) throws Seman ogw.startWalking(topNodes, null); } + private void runDynPatitionSortOptimizations(OptimizeSparkProcContext procCtx) throws SemanticException { + // run Sorted dynamic partition optimization + HiveConf hConf = procCtx.getConf(); + ParseContext parseContext = procCtx.getParseContext(); + runDynPatitionSortOptimizations(parseContext,hConf); + } + /** * TODO: need to turn on rules that's commented out and add more if necessary. */ diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out index e06624c32b91b22d3dfeefd3c6d7c8cc95ae92c1..e1af88ad8128567de3ac2697e5d1cc1379b838ec 100644 --- a/ql/src/test/results/clientpositive/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -97,7 +97,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4542 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -184,7 +184,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4542 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -235,7 +235,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4542 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -331,7 +331,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 9085 + totalSize 8126 #### A masked pattern was here #### # Storage Information @@ -380,7 +380,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 9085 + totalSize 8126 #### A masked pattern was here #### # Storage Information @@ -455,11 +455,11 @@ STAGE PLANS: TableScan alias: acid filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(key) mode: hash @@ -591,7 +591,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 176000 - totalSize 3485 + totalSize 3008 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out new file mode 100644 index 0000000000000000000000000000000000000000..427f81957c84e382481e858a94fffa99d87216d2 --- /dev/null +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out @@ -0,0 +1,3797 @@ +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part +POSTHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part +PREHOOK: query: create table over1k_part_limit like over1k_part +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_limit +POSTHOOK: query: create table over1k_part_limit like over1k_part +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_limit +PREHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck +PREHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_limit + +PREHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort + +PREHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_limit + +PREHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck + +PREHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort + +PREHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 14 + rawDataSize 362 + totalSize 376 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select count(*) from over1k_part +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part +PREHOOK: Input: default@over1k_part@ds=foo/t=27 +PREHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part +POSTHOOK: Input: default@over1k_part@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_limit +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_limit +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_limit +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_limit +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +20 +PREHOOK: query: select count(*) from over1k_part_buck +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck +PREHOOK: Input: default@over1k_part_buck@t=27 +PREHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck +POSTHOOK: Input: default@over1k_part_buck@t=27 +POSTHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_buck_sort +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort +PREHOOK: Input: default@over1k_part_buck_sort@t=27 +PREHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: Input: default@over1k_part_buck_sort@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part2 +POSTHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part2 +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = 27Y) or _col0 is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort2 +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, t + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort2 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort2 + +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + outputColumnNames: si, b, f, s, t, i + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: s (type: string), t (type: tinyint), i (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint), _col2 (type: int) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint), KEY._col2 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_part3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_part3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: drop table over1k_part3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: analyze table over1k_n3 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +POSTHOOK: query: analyze table over1k_n3 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 352 Data size: 39072 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 5bc944271d023c725c776ba204b2bfd834659cb7..6c71200d7bd70406b1dbb2882390d80c08af1c83 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -31,10 +31,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4 - Stage-4 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -48,62 +46,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col2 (type: string), _col3 (type: string), _bucket_number (type: string), _col1 (type: string) + sort order: ++++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number File Output Operator compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), UDFToInteger(_col3) (type: int) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Move Operator @@ -118,7 +78,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Stage: Stage-3 + Stage: Stage-2 Stats Work Basic Stats Work: Column Stats Desc: @@ -126,33 +86,3 @@ STAGE PLANS: Column Types: string, string Table: default.non_acid - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - diff --git a/ql/src/test/results/clientpositive/spark/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/spark/dynpart_sort_optimization.q.out new file mode 100644 index 0000000000000000000000000000000000000000..ddc5106bf572877b45f061542123e52912931b62 --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/dynpart_sort_optimization.q.out @@ -0,0 +1,3504 @@ +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part +POSTHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part +PREHOOK: query: create table over1k_part_limit like over1k_part +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_limit +POSTHOOK: query: create table over1k_part_limit like over1k_part +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_limit +PREHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck +PREHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 14 + rawDataSize 362 + totalSize 376 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select count(*) from over1k_part +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part +PREHOOK: Input: default@over1k_part@ds=foo/t=27 +PREHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part +POSTHOOK: Input: default@over1k_part@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_limit +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_limit +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_limit +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_limit +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +20 +PREHOOK: query: select count(*) from over1k_part_buck +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck +PREHOOK: Input: default@over1k_part_buck@t=27 +PREHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck +POSTHOOK: Input: default@over1k_part_buck@t=27 +POSTHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_buck_sort +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort +PREHOOK: Input: default@over1k_part_buck_sort@t=27 +PREHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: Input: default@over1k_part_buck_sort@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part2 +POSTHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part2 +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = 27Y) or _col0 is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort2 +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_part3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_part3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: drop table over1k_part3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: analyze table over1k_n3 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +POSTHOOK: query: analyze table over1k_n3 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3