diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index 80dbbee5bad6d99144be9cfac8368db9cf9a7b36..a1746fcad8ae78a5c73be84f032607a1e6ce00c9 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -84,8 +84,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -108,12 +107,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - null sort order: - sort order: - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col1 (type: string), _bucket_number (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col1 (type: string) tag: -1 - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -159,13 +158,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _bucket_number File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### + Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### @@ -194,34 +193,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: id, key - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(id, 'hll') - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Move Operator @@ -263,83 +234,6 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: struct) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10002 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index a8cdc8fe057b8725a4d676dee434723fc8899b0a..ba3752d6a6e3383e4ac77bdc61363f032e39d0b4 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -102,8 +102,7 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0, Stage-3 - Stage-3 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -126,12 +125,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - null sort order: - sort order: - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + key expressions: _col1 (type: string), _bucket_number (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col1 (type: string) tag: -1 - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -177,13 +176,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _bucket_number File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### + Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### @@ -212,34 +211,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Select Operator - expressions: _col0 (type: int), _col1 (type: string) - outputColumnNames: id, key - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(id, 'hll') - keys: key (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-0 Move Operator @@ -281,83 +252,6 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - GatherStats: false - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE - tag: -1 - value expressions: _col1 (type: struct) - auto parallelism: false - Execution mode: vectorized - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### - Partition - base file name: -mr-10002 - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1 - columns.types string,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Truncated Path -> Alias: -#### A masked pattern was here #### - Needs Tagging: false - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1 - columns.types struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out index 826fae9d422d3423f8ab32997348ce246af11938..7b2561358a95285fbfaaa6d196e35a0bea62ce85 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +200 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +50 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +100 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +300 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +350 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -50 +400 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -50 +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out index bb63070503051415b5e947f02dfe4c3660feb955..1201ce2107cdd428eb481f48ea5444a3ac71d68b 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -0 +200 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -100 +300 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -200 +400 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -300 +500 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out index 7758dc9d53a7872818ed16097c7524be3e11f83e..0931e3d5576c87ee918e1a86502e2a208e976513 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +200 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +50 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +100 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +300 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +350 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -50 +400 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -50 +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out index 0ccd0e43452c7f1826524d61f29792f165560edd..15ae3d91a75ca8cbe3d7e036ac2ff7eef87ec104 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -0 +200 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -100 +300 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -200 +400 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -300 +500 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out index 340791aa7ffeff777c28be2335b2733263afa1da..24fc5258043465cc6a00a271ca3e797884063aaf 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +200 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +50 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +100 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +300 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +350 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -50 +400 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -50 +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out index 160842236b55c2cef4ed211708ac89e167bb48f4..6bcfe41262dee389e854aa399dcc854a2b16599b 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -0 +200 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -100 +300 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -200 +400 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fb50588fe7ce06df6bf41775dbef42143d3d8bb9..141d09351e5a01cf8a49952b37da65ba75dba2d5 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1138,6 +1138,7 @@ spark.query.files=add_part_multiple.q, \ decimal_1_1.q, \ decimal_join.q, \ disable_merge_for_bucketing.q, \ + dynpart_sort_optimization.q, \ enforce_order.q, \ escape_clusterby1.q, \ escape_distributeby1.q, \ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index ff35e8c082088fc93bed996d83d64e9892f20f2a..2892a7f8f69d679d39f05bf5a8219402829b47ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; +import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.orc.OrcConf; @@ -200,7 +201,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, destTable.getCols()); List sortPositions = null; List sortOrder = null; - ArrayList bucketColumns; + ArrayList bucketColumns = null; if (fsOp.getConf().getWriteType() == AcidUtils.Operation.UPDATE || fsOp.getConf().getWriteType() == AcidUtils.Operation.DELETE) { // When doing updates and deletes we always want to sort on the rowid because the ACID @@ -208,7 +209,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // ignore whatever comes from the table and enforce this sort order instead. sortPositions = Collections.singletonList(0); sortOrder = Collections.singletonList(1); // 1 means asc, could really use enum here in the thrift if - bucketColumns = new ArrayList<>(); /** * ROW__ID is always the 1st column of Insert representing Update/Delete operation * (set up in {@link org.apache.hadoop.hive.ql.parse.UpdateDeleteSemanticAnalyzer}) @@ -219,9 +219,13 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, ColumnInfo ci = fsParent.getSchema().getSignature().get(0); if (!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) { throw new IllegalStateException("expected 1st column to be ROW__ID but got wrong type: " + ci.toString()); + } + + if (numBuckets > 0) { + bucketColumns = new ArrayList<>(); + //add a cast(ROW__ID as int) to wrap in UDFToInteger() + bucketColumns.add(ParseUtils.createConversionCast(new ExprNodeColumnDesc(ci), TypeInfoFactory.intTypeInfo)); } - //add a cast(ROW__ID as int) to wrap in UDFToInteger() - bucketColumns.add(ParseUtils.createConversionCast(new ExprNodeColumnDesc(ci), TypeInfoFactory.intTypeInfo)); } else { if (!destTable.getSortCols().isEmpty()) { // Sort columns specified by table @@ -281,7 +285,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } } RowSchema selRS = new RowSchema(fsParent.getSchema()); - if (!bucketColumns.isEmpty()) { + if (bucketColumns!= null && !bucketColumns.isEmpty()) { descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, ReduceField.KEY.toString()+"."+BUCKET_NUMBER_COL_NAME, null, false)); colNames.add(BUCKET_NUMBER_COL_NAME); @@ -304,7 +308,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // Set if partition sorted or partition bucket sorted fsOp.getConf().setDpSortState(FileSinkDesc.DPSortState.PARTITION_SORTED); - if (!bucketColumns.isEmpty()) { + if (bucketColumns!=null && !bucketColumns.isEmpty()) { fsOp.getConf().setDpSortState(FileSinkDesc.DPSortState.PARTITION_BUCKET_SORTED); } @@ -485,7 +489,7 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, int numPartAndBuck = partitionPositions.size(); keyColsPosInVal.addAll(partitionPositions); - if (!bucketColumns.isEmpty()) { + if (bucketColumns != null && !bucketColumns.isEmpty()) { keyColsPosInVal.add(-1); numPartAndBuck += 1; } @@ -706,7 +710,13 @@ private boolean shouldDo(List partitionPos, Operator colStats = fsParent.getStatistics().getColumnStats(); + Statistics tStats = fsParent.getStatistics(); + if (tStats == null) { + return true; + } + + + List colStats = tStats.getColumnStats(); if (colStats == null || colStats.isEmpty()) { return true; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java index 302b3506f361fe72283257b7ec7d8740938c4a0c..76ebcf8e94f7249afb5c79f5b804f389116e76a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java @@ -118,6 +118,12 @@ protected void setInputFormat(Task task) { } } + @Override + protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, + Set outputs) throws SemanticException { + this.runDynPatitionSortOptimizations(pCtx, conf); + } + private void setInputFormat(MapWork work, Operator op) { if (op.isUseBucketizedHiveInputFormat()) { work.setUseBucketizedHiveInputFormat(true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index baefe1b39eb1936f8a5dd1e2be5bbf3bfb73d63b..7f99b42777bc5ca07fe1beca3464d3b084a71ffa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -48,6 +48,9 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; +import org.apache.hadoop.hive.ql.optimizer.NonBlockingOpDeDupProc; +import org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.plan.BasicStatsWork; @@ -65,7 +68,6 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; -import org.apache.hadoop.hive.ql.stats.BasicStatsNoJobTask; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.DefaultFetchFormatter; import org.apache.hadoop.hive.serde2.NoOpFetchFormatter; @@ -654,6 +656,30 @@ protected abstract void optimizeTaskPlan(List> root protected abstract void generateTaskTree(List> rootTasks, ParseContext pCtx, List> mvTask, Set inputs, Set outputs) throws SemanticException; + /* + * Called for dynamic partitioning sort optimization. + */ + protected void runDynPatitionSortOptimizations(ParseContext parseContext, HiveConf hConf) throws SemanticException { + // run Sorted dynamic partition optimization + + if(HiveConf.getBoolVar(hConf, HiveConf.ConfVars.DYNAMICPARTITIONING) && + HiveConf.getVar(hConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") && + !HiveConf.getBoolVar(hConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { + new SortedDynPartitionOptimizer().transform(parseContext); + + if(HiveConf.getBoolVar(hConf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION) + || parseContext.hasAcidWrite()) { + + // Dynamic sort partition adds an extra RS therefore need to de-dup + new ReduceSinkDeDuplication().transform(parseContext); + // there is an issue with dedup logic wherein SELECT is created with wrong columns + // NonBlockingOpDeDupProc fixes that + new NonBlockingOpDeDupProc().transform(parseContext); + + } + } + } + /** * Create a clone of the parse context */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 0a76ffa28bd9476d2a14083d2ef4522fa46c3868..9647a5824e20b90a50960e9f4fcd65777cd47705 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -120,6 +120,9 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // Annotation OP tree with statistics runStatsAnnotation(procCtx); + // Run Dynamic Partitioning sort Optimization. + runDynPatitionSortOptimizations(procCtx); + // Set reducer parallelism runSetReducerParallelism(procCtx); @@ -336,6 +339,13 @@ private void runJoinOptimizations(OptimizeSparkProcContext procCtx) throws Seman ogw.startWalking(topNodes, null); } + private void runDynPatitionSortOptimizations(OptimizeSparkProcContext procCtx) throws SemanticException { + // run Sorted dynamic partition optimization + HiveConf hConf = procCtx.getConf(); + ParseContext parseContext = procCtx.getParseContext(); + runDynPatitionSortOptimizations(parseContext, hConf); + } + /** * TODO: need to turn on rules that's commented out and add more if necessary. */ diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index c74a707e4bac80850858f0ee89c54fb076df8bc8..6972c508c79697329f2dd40f12062ddebc57e7d1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -764,6 +764,7 @@ private void checkExpected(List rs, String[][] expected, String msg) { @Test public void testCompactStatsGather() throws Exception { hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); + hiveConf.setIntVar(HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD, -1); runStatementOnDriver("drop table if exists T"); runStatementOnDriver("create table T(a int, b int) partitioned by (p int, q int) " + "stored as orc TBLPROPERTIES ('transactional'='true')"); diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out index e06624c32b91b22d3dfeefd3c6d7c8cc95ae92c1..e1af88ad8128567de3ac2697e5d1cc1379b838ec 100644 --- a/ql/src/test/results/clientpositive/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -97,7 +97,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4542 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -184,7 +184,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4542 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -235,7 +235,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4542 + totalSize 4063 #### A masked pattern was here #### # Storage Information @@ -331,7 +331,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 9085 + totalSize 8126 #### A masked pattern was here #### # Storage Information @@ -380,7 +380,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 9085 + totalSize 8126 #### A masked pattern was here #### # Storage Information @@ -455,11 +455,11 @@ STAGE PLANS: TableScan alias: acid filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(key) mode: hash @@ -591,7 +591,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 176000 - totalSize 3485 + totalSize 3008 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/annotate_stats_part.q.out b/ql/src/test/results/clientpositive/annotate_stats_part.q.out index 04320a7ff27c703c1a3e6d4ef9ae4be50cc5ac3c..1b9881194294ffb41cbf6eb1fd35ddaf71e729ab 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_part.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_part.q.out @@ -102,11 +102,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc_n4 - Statistics: Num rows: 20 Data size: 15680 Basic stats: PARTIAL Column stats: PARTIAL + Statistics: Num rows: 20 Data size: 15670 Basic stats: PARTIAL Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 20 Data size: 15680 Basic stats: PARTIAL Column stats: PARTIAL + Statistics: Num rows: 20 Data size: 15670 Basic stats: PARTIAL Column stats: PARTIAL ListSink PREHOOK: query: analyze table loc_orc_n4 partition(year='2001') compute statistics diff --git a/ql/src/test/results/clientpositive/autoColumnStats_1.q.out b/ql/src/test/results/clientpositive/autoColumnStats_1.q.out index 4131535972447c147d5d2cf46bd556d727ab5eda..bcabc02a4b9229547ea26287223fe85b9605fe25 100644 --- a/ql/src/test/results/clientpositive/autoColumnStats_1.q.out +++ b/ql/src/test/results/clientpositive/autoColumnStats_1.q.out @@ -1389,11 +1389,11 @@ STAGE PLANS: Processor Tree: TableScan alias: a_n12 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select value from b_n9 @@ -1416,11 +1416,11 @@ STAGE PLANS: Processor Tree: TableScan alias: b_n9 - Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select key from b_n9 @@ -1443,11 +1443,11 @@ STAGE PLANS: Processor Tree: TableScan alias: b_n9 - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select value from c_n2 @@ -1472,11 +1472,11 @@ STAGE PLANS: Processor Tree: TableScan alias: c_n2 - Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select key from c_n2 @@ -1501,10 +1501,10 @@ STAGE PLANS: Processor Tree: TableScan alias: c_n2 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/autoColumnStats_2.q.out b/ql/src/test/results/clientpositive/autoColumnStats_2.q.out index a8371236e7df260fb4aed0de9198ff81330b7b14..3618b02f8e59ed4fd4828cc366a9211b8dbd5ff7 100644 --- a/ql/src/test/results/clientpositive/autoColumnStats_2.q.out +++ b/ql/src/test/results/clientpositive/autoColumnStats_2.q.out @@ -1519,11 +1519,11 @@ STAGE PLANS: Processor Tree: TableScan alias: a_n3 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select value from b_n3 @@ -1546,11 +1546,11 @@ STAGE PLANS: Processor Tree: TableScan alias: b_n3 - Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select key from b_n3 @@ -1573,11 +1573,11 @@ STAGE PLANS: Processor Tree: TableScan alias: b_n3 - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 87000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 185608 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select value from c_n1 @@ -1602,11 +1602,11 @@ STAGE PLANS: Processor Tree: TableScan alias: c_n1 - Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: value (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: explain select key from c_n1 @@ -1631,10 +1631,10 @@ STAGE PLANS: Processor Tree: TableScan alias: c_n1 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 371032 Basic stats: COMPLETE Column stats: NONE ListSink diff --git a/ql/src/test/results/clientpositive/autoColumnStats_6.q.out b/ql/src/test/results/clientpositive/autoColumnStats_6.q.out index 602e6dbb891295fb229703d1e2543735527f56b1..2c2baf1f0a8ae7abe6575c15d5030bf3c7b38739 100644 --- a/ql/src/test/results/clientpositive/autoColumnStats_6.q.out +++ b/ql/src/test/results/clientpositive/autoColumnStats_6.q.out @@ -29,13 +29,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -48,57 +43,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge2a - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) - outputColumnNames: key, value, one, two, three - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: one (type: string), two (type: string), three (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: struct), _col4 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a Stage: Stage-0 Move Operator @@ -122,26 +85,6 @@ STAGE PLANS: Column Types: int, string Table: default.orcfile_merge2a - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, PMOD(HASH(value), 10) as three diff --git a/ql/src/test/results/clientpositive/autoColumnStats_8.q.out b/ql/src/test/results/clientpositive/autoColumnStats_8.q.out index 492acd284d4123383a86bcc44c6cee51741ba4ca..d0c660219ff792811217d3dafb07cbd1f38a7b1d 100644 --- a/ql/src/test/results/clientpositive/autoColumnStats_8.q.out +++ b/ql/src/test/results/clientpositive/autoColumnStats_8.q.out @@ -59,9 +59,9 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-1, Stage-5 - Stage-5 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -79,54 +79,14 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col2 (type: struct), _col3 (type: struct) - auto parallelism: false + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false Filter Operator isSamplingPred: false predicate: (ds > '2008-04-08') (type: boolean) @@ -137,62 +97,23 @@ STAGE PLANS: Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 2 + GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Static Partition Specification: ds=2008-12-31/ - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - bucket_count -1 column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8 + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TotalFiles: 1 - GatherStats: true + GatherStats: false MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: '2008-12-31' (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,struct,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Execution mode: vectorized Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -399,38 +320,39 @@ STAGE PLANS: /srcpart/ds=2008-04-09/hr=12 [srcpart] Needs Tagging: false Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE + name default.nzhang_part8 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types struct:struct:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Stage: Stage-0 Move Operator @@ -465,57 +387,18 @@ STAGE PLANS: Basic Stats Work: #### A masked pattern was here #### - Stage: Stage-1 - Move Operator - tables: - partition: - ds 2008-12-31 - hr - replace: true -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8 - Stage: Stage-4 - Stats Work - Basic Stats Work: -#### A masked pattern was here #### - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.nzhang_part8 - Is Table Level Stats: false - - Stage: Stage-5 Map Reduce Map Operator Tree: TableScan GatherStats: false Reduce Output Operator - key expressions: '2008-12-31' (type: string), _col1 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: '2008-12-31' (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) tag: -1 - value expressions: _col2 (type: struct), _col3 (type: struct) + value expressions: _col0 (type: string), _col1 (type: string) auto parallelism: false Execution mode: vectorized Path -> Alias: @@ -528,8 +411,8 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,struct,struct + columns _col0,_col1,_col2 + columns.types string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -538,8 +421,8 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,struct,struct + columns _col0,_col1,_col2 + columns.types string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -547,38 +430,78 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: '2008-12-31' (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), '2008-12-31' (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 +#### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2008-12-31/ + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types struct:struct:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + name default.nzhang_part8 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-1 + Move Operator + tables: + partition: + ds 2008-12-31 + hr + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.nzhang_part8 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8 + + Stage: Stage-5 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.nzhang_part8 + Is Table Level Stats: false PREHOOK: query: from srcpart insert overwrite table nzhang_part8 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git a/ql/src/test/results/clientpositive/dynamic_partition_insert.q.out b/ql/src/test/results/clientpositive/dynamic_partition_insert.q.out index 15b7460c0305352544498b2593c3e445943b358a..2fab374433058bb5823247de686b1ac0e2ad594d 100644 --- a/ql/src/test/results/clientpositive/dynamic_partition_insert.q.out +++ b/ql/src/test/results/clientpositive/dynamic_partition_insert.q.out @@ -1100,8 +1100,8 @@ John Doe 23 USA CA NULL NULL USA CA Jane Doe 22 USA TX NULL NULL USA TX -John Doe 23 USA __HIVE_DEFAULT_PARTITION__ Jane Doe 22 USA __HIVE_DEFAULT_PARTITION__ +John Doe 23 USA __HIVE_DEFAULT_PARTITION__ PREHOOK: query: INSERT INTO TABLE table2_n10( name, country) SELECT name, country FROM table1_n15 PREHOOK: type: QUERY PREHOOK: Input: default@table1_n15 @@ -1143,10 +1143,10 @@ John Doe 23 USA CA NULL NULL USA CA Jane Doe 22 USA TX NULL NULL USA TX -John Doe 23 USA __HIVE_DEFAULT_PARTITION__ Jane Doe 22 USA __HIVE_DEFAULT_PARTITION__ -John Doe NULL USA __HIVE_DEFAULT_PARTITION__ +John Doe 23 USA __HIVE_DEFAULT_PARTITION__ Jane Doe NULL USA __HIVE_DEFAULT_PARTITION__ +John Doe NULL USA __HIVE_DEFAULT_PARTITION__ PREHOOK: query: DROP TABLE table2_n10 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table2_n10 diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out new file mode 100644 index 0000000000000000000000000000000000000000..427f81957c84e382481e858a94fffa99d87216d2 --- /dev/null +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out @@ -0,0 +1,3797 @@ +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part +POSTHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part +PREHOOK: query: create table over1k_part_limit like over1k_part +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_limit +POSTHOOK: query: create table over1k_part_limit like over1k_part +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_limit +PREHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck +PREHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_limit + +PREHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort + +PREHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_limit + +PREHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck + +PREHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort + +PREHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 14 + rawDataSize 362 + totalSize 376 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select count(*) from over1k_part +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part +PREHOOK: Input: default@over1k_part@ds=foo/t=27 +PREHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part +POSTHOOK: Input: default@over1k_part@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_limit +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_limit +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_limit +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_limit +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +20 +PREHOOK: query: select count(*) from over1k_part_buck +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck +PREHOOK: Input: default@over1k_part_buck@t=27 +PREHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck +POSTHOOK: Input: default@over1k_part_buck@t=27 +POSTHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_buck_sort +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort +PREHOOK: Input: default@over1k_part_buck_sort@t=27 +PREHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: Input: default@over1k_part_buck_sort@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part2 +POSTHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part2 +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = 27Y) or _col0 is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part2 + +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort2 +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, t + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort2 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: tinyint) + sort order: + + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part_buck_sort2 + +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"b\":\"true\",\"f\":\"true\",\"i\":\"true\",\"si\":\"true\"}} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + outputColumnNames: si, b, f, s, t, i + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: s (type: string), t (type: tinyint), i (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint), _col2 (type: int) + sort order: +++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint), _col2 (type: int) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint), KEY._col2 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint), _col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, b, f + Column Types: smallint, bigint, float + Table: default.over1k_part3 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_part3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_part3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: drop table over1k_part3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: analyze table over1k_n3 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +POSTHOOK: query: analyze table over1k_n3 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-2 depends on stages: Stage-0 + Stage-3 + Stage-5 + Stage-6 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 352 Data size: 39072 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-7 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-6 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 5bc944271d023c725c776ba204b2bfd834659cb7..6c71200d7bd70406b1dbb2882390d80c08af1c83 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -31,10 +31,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0, Stage-4 - Stage-4 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 @@ -48,62 +46,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) + key expressions: _col2 (type: string), _col3 (type: string), _bucket_number (type: string), _col1 (type: string) + sort order: ++++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - sort order: - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - Execution mode: vectorized - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number File Output Operator compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), UDFToInteger(_col3) (type: int) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: int) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Move Operator @@ -118,7 +78,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Stage: Stage-3 + Stage: Stage-2 Stats Work Basic Stats Work: Column Stats Desc: @@ -126,33 +86,3 @@ STAGE PLANS: Column Types: string, string Table: default.non_acid - Stage: Stage-4 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: int) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - diff --git a/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out b/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out index ad67fb852fa2af9aecccfcd24ebb8adbed8fb132..a9b927b7f348743033ce9b7d632ee4e7cd6ea11a 100644 --- a/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out +++ b/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out @@ -267,7 +267,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 426 + totalSize 425 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -313,7 +313,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 433 + totalSize 432 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -340,12 +340,12 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc_1d_n1 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Select Operator expressions: state (type: string) outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: explain extended select state,locid from loc_orc_1d_n1 @@ -488,7 +488,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 426 + totalSize 425 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -534,7 +534,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 433 + totalSize 432 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -561,12 +561,12 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc_1d_n1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Select Operator expressions: state (type: string), locid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: analyze table loc_orc_1d_n1 partition(year='2000') compute statistics for columns state @@ -737,7 +737,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 426 + totalSize 425 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -783,7 +783,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 433 + totalSize 432 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -958,7 +958,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 426 + totalSize 425 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -1004,7 +1004,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n1 { string state, i32 locid, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 433 + totalSize 432 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -1031,12 +1031,12 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc_1d_n1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Select Operator expressions: state (type: string), locid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: create table if not exists loc_orc_2d_n1 ( @@ -1681,12 +1681,12 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc_2d_n1 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Select Operator expressions: state (type: string) outputColumnNames: _col0 - Statistics: Num rows: 20 Data size: 1740 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: explain extended select state,locid from loc_orc_2d_n1 @@ -2249,11 +2249,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc_2d_n1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Select Operator expressions: state (type: string), locid (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 20 Data size: 1820 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 1840 Basic stats: COMPLETE Column stats: PARTIAL ListSink diff --git a/ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out b/ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out index 2dc7b6346d0bfccaeb1cb19d387d33ebeea63143..fc782b4590860ea98a9ff596ceac454c606d15f1 100644 --- a/ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out +++ b/ql/src/test/results/clientpositive/implicit_cast_during_insert.q.out @@ -43,44 +43,22 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) - sort order: + + sort order: ++ Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + value expressions: UDFToInteger(_col2) (type: int), _col1 (type: string) Reduce Operator Tree: Select Operator - expressions: UDFToInteger(KEY.reducesinkkey0) (type: int), VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.implicit_cast_during_insert - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: c1, c2, p1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(c1, 'hll'), compute_stats(c2, 'hll') - keys: p1 (type: string) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out b/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out index a7034a6d33f273b789a457b020cb3273ff0ec695..f865eb96e058bfc19afb7699b3a387909d561f72 100644 --- a/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out +++ b/ql/src/test/results/clientpositive/infer_bucket_sort_dyn_part.q.out @@ -197,7 +197,7 @@ Partition Parameters: numFiles 1 numRows 309 rawDataSize 1173 - totalSize 1342 + totalSize 1381 #### A masked pattern was here #### # Storage Information @@ -260,7 +260,7 @@ Partition Parameters: numFiles 1 numRows 155 rawDataSize 586 - totalSize 719 + totalSize 722 #### A masked pattern was here #### # Storage Information @@ -268,9 +268,9 @@ SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat Compressed: No -Num Buckets: 1 -Bucket Columns: [key] -Sort Columns: [Order(col:key, order:1)] +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] Storage Desc Params: serialization.format 1 PREHOOK: query: DESCRIBE FORMATTED test_table_n8 PARTITION (ds='2008-04-08', hr='12') @@ -298,7 +298,7 @@ Partition Parameters: numFiles 1 numRows 154 rawDataSize 591 - totalSize 722 + totalSize 741 #### A masked pattern was here #### # Storage Information @@ -306,9 +306,9 @@ SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat Compressed: No -Num Buckets: 1 -Bucket Columns: [key] -Sort Columns: [Order(col:key, order:1)] +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] Storage Desc Params: serialization.format 1 PREHOOK: query: CREATE TABLE srcpart_merge_dp_n3 LIKE srcpart @@ -419,14 +419,14 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 - Stage-2 depends on stages: Stage-0, Stage-8 - Stage-3 + Stage-2 depends on stages: Stage-1 + Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 Stage-5 - Stage-6 depends on stages: Stage-5 - Stage-8 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 + Stage-3 depends on stages: Stage-0 + Stage-4 + Stage-6 + Stage-7 depends on stages: Stage-6 STAGE PLANS: Stage: Stage-1 @@ -466,33 +466,39 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.test_table_n8 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-7 + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.test_table_n8 + + Stage: Stage-8 Conditional Operator - Stage: Stage-4 + Stage: Stage-5 Move Operator files: hdfs directory: true @@ -511,7 +517,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.test_table_n8 - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -519,56 +525,26 @@ STAGE PLANS: Column Types: string, string Table: default.test_table_n8 - Stage: Stage-3 + Stage: Stage-4 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-5 + Stage: Stage-6 Merge File Operator Map Operator Tree: RCFile Merge Operator merge level: block input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - Stage: Stage-6 + Stage: Stage-7 Move Operator files: hdfs directory: true #### A masked pattern was here #### - Stage: Stage-8 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: INSERT OVERWRITE TABLE test_table_n8 PARTITION (ds = '2008-04-08', hr) SELECT key, value, IF (key % 100 == 0, '11', '12') FROM (SELECT key, COUNT(*) AS value FROM srcpart @@ -619,7 +595,7 @@ Partition Parameters: numFiles 1 numRows 4 rawDataSize 14 - totalSize 115 + totalSize 94 #### A masked pattern was here #### # Storage Information @@ -654,10 +630,10 @@ Table: test_table_n8 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} - numFiles 2 + numFiles 1 numRows 305 rawDataSize 1163 - totalSize 1427 + totalSize 1346 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/insert_into6.q.out b/ql/src/test/results/clientpositive/insert_into6.q.out index 880bda687e71970bfbe3d29cdd9576d5dac75d5b..f13f764b0f9238532dcc6b93bc4e706be56ff6d6 100644 --- a/ql/src/test/results/clientpositive/insert_into6.q.out +++ b/ql/src/test/results/clientpositive/insert_into6.q.out @@ -196,13 +196,8 @@ POSTHOOK: Input: default@insert_into6a@ds=1 POSTHOOK: Input: default@insert_into6a@ds=2 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -215,57 +210,25 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2680 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2680 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 250 Data size: 2680 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 250 Data size: 2680 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2680 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct), _col2 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 125 Data size: 1340 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 125 Data size: 1340 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 125 Data size: 1340 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 250 Data size: 2680 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.insert_into6b Stage: Stage-0 Move Operator @@ -287,36 +250,6 @@ STAGE PLANS: Column Types: int, string Table: default.insert_into6b - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.insert_into6b - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: INSERT INTO TABLE insert_into6b PARTITION (ds) SELECT * FROM insert_into6a PREHOOK: type: QUERY PREHOOK: Input: default@insert_into6a diff --git a/ql/src/test/results/clientpositive/load_dyn_part1.q.out b/ql/src/test/results/clientpositive/load_dyn_part1.q.out index 63a17f47b6dff695f5fc9582071c00addfd6a50f..ae0779cb01b84ef52695a259e0d5619ffff38109 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part1.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part1.q.out @@ -65,16 +65,11 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0, Stage-10 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-1 depends on stages: Stage-2 - Stage-9 depends on stages: Stage-1, Stage-10 - Stage-10 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -90,30 +85,11 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1_n0 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Filter Operator predicate: (ds > '2008-04-08') (type: boolean) Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE @@ -123,55 +99,24 @@ STAGE PLANS: Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part2_n0 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: '2008-12-31' (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part1_n0 Stage: Stage-0 Move Operator @@ -198,31 +143,25 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1_n0 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1_n0 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part2_n0 Stage: Stage-1 Move Operator @@ -237,7 +176,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part2_n0 - Stage: Stage-9 + Stage: Stage-5 Stats Work Basic Stats Work: Column Stats Desc: @@ -245,36 +184,6 @@ STAGE PLANS: Column Types: string, string Table: default.nzhang_part2_n0 - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: '2008-12-31' (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: '2008-12-31' (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: '2008-12-31' (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), '2008-12-31' (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: from srcpart insert overwrite table nzhang_part1_n0 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' insert overwrite table nzhang_part2_n0 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' diff --git a/ql/src/test/results/clientpositive/load_dyn_part10.q.out b/ql/src/test/results/clientpositive/load_dyn_part10.q.out index 0a9df77c5209ba9827d8aa1da8399b36e152d076..c6d9b6c4ff0a76b05cf8fedfa160dd47d885bf11 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part10.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part10.q.out @@ -65,48 +65,25 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part10 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-12-31' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part10 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/load_dyn_part3.q.out b/ql/src/test/results/clientpositive/load_dyn_part3.q.out index dc94977b8bd9a9c1cf91a05ddeadf532d3721779..ebf3c52f9336e6a59a90c2a5aad6bb8190a5ad82 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part3.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part3.q.out @@ -67,48 +67,25 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part3 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part3 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/load_dyn_part4.q.out b/ql/src/test/results/clientpositive/load_dyn_part4.q.out index 533570ee3390b78361d4f6b139ef087e2a2e88fc..e30d6aec51f36656f13654411d45512abafb6ec1 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part4.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part4.q.out @@ -77,48 +77,25 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part4 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part4 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/load_dyn_part8.q.out b/ql/src/test/results/clientpositive/load_dyn_part8.q.out index 97b8886bd3b0d8121d9cc146596c1d15e1a2566d..1037a339d1d4b0520c6ce46dca34022831f26278 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part8.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part8.q.out @@ -59,9 +59,9 @@ STAGE DEPENDENCIES: Stage-2 is a root stage Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 - Stage-1 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-1, Stage-5 - Stage-5 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -79,54 +79,14 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8_n0 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8_n0 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8_n0 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col2 (type: struct), _col3 (type: struct) - auto parallelism: false + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false Filter Operator isSamplingPred: false predicate: (ds > '2008-04-08') (type: boolean) @@ -137,62 +97,23 @@ STAGE PLANS: Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - GlobalTableId: 2 + GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Static Partition Specification: ds=2008-12-31/ - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: - bucket_count -1 column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8_n0 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8_n0 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8_n0 + columns _col0,_col1,_col2 + columns.types string,string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe TotalFiles: 1 - GatherStats: true + GatherStats: false MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: '2008-12-31' (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,struct,struct - escape.delim \ - serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + Execution mode: vectorized Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -399,38 +320,39 @@ STAGE PLANS: /srcpart/ds=2008-04-09/hr=12 [srcpart] Needs Tagging: false Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE + name default.nzhang_part8_n0 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8_n0 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types struct:struct:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8_n0 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Stage: Stage-0 Move Operator @@ -465,57 +387,18 @@ STAGE PLANS: Basic Stats Work: #### A masked pattern was here #### - Stage: Stage-1 - Move Operator - tables: - partition: - ds 2008-12-31 - hr - replace: true -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8_n0 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8_n0 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8_n0 - Stage: Stage-4 - Stats Work - Basic Stats Work: -#### A masked pattern was here #### - Column Stats Desc: - Columns: key, value - Column Types: string, string - Table: default.nzhang_part8_n0 - Is Table Level Stats: false - - Stage: Stage-5 Map Reduce Map Operator Tree: TableScan GatherStats: false Reduce Output Operator - key expressions: '2008-12-31' (type: string), _col1 (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: '2008-12-31' (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) tag: -1 - value expressions: _col2 (type: struct), _col3 (type: struct) + value expressions: _col0 (type: string), _col1 (type: string) auto parallelism: false Execution mode: vectorized Path -> Alias: @@ -528,8 +411,8 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,struct,struct + columns _col0,_col1,_col2 + columns.types string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -538,8 +421,8 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat properties: column.name.delimiter , - columns _col0,_col1,_col2,_col3 - columns.types string,string,struct,struct + columns _col0,_col1,_col2 + columns.types string,string,string escape.delim \ serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe @@ -547,38 +430,78 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: '2008-12-31' (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), '2008-12-31' (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 +#### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2008-12-31/ + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types struct:struct:string:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + name default.nzhang_part8_n0 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8_n0 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8_n0 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-1 + Move Operator + tables: + partition: + ds 2008-12-31 + hr + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.nzhang_part8_n0 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8_n0 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8_n0 + + Stage: Stage-5 + Stats Work + Basic Stats Work: +#### A masked pattern was here #### + Column Stats Desc: + Columns: key, value + Column Types: string, string + Table: default.nzhang_part8_n0 + Is Table Level Stats: false PREHOOK: query: from srcpart insert overwrite table nzhang_part8_n0 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' diff --git a/ql/src/test/results/clientpositive/load_dyn_part9.q.out b/ql/src/test/results/clientpositive/load_dyn_part9.q.out index 34b1f26c815d6787532ebf37d6d4ff82a93854ff..4e272bbe6b6f096a0611591619d3a1096c342dc0 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part9.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part9.q.out @@ -65,48 +65,25 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part9 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part9 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/merge3.q.out b/ql/src/test/results/clientpositive/merge3.q.out index 5a4747a624318960de456df1d2da57a9b7cc2ce6..0b4da94db185003f426246f8e7ae7efdda435b0a 100644 --- a/ql/src/test/results/clientpositive/merge3.q.out +++ b/ql/src/test/results/clientpositive/merge3.q.out @@ -2415,54 +2415,15 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments - columns.types string:string -#### A masked pattern was here #### - name default.merge_src_part2 - partition_columns ds - partition_columns.types string - serialization.ddl struct merge_src_part2 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_src_part2 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - tag: -1 - value expressions: _col1 (type: struct), _col2 (type: struct) - auto parallelism: false + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Execution mode: vectorized Path -> Alias: #### A masked pattern was here #### Path -> Partition: @@ -2482,7 +2443,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.merge_src_part - numFiles 2 + numFiles 1 numRows 1000 partition_columns ds partition_columns.types string @@ -2530,7 +2491,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.merge_src_part - numFiles 2 + numFiles 1 numRows 1000 partition_columns ds partition_columns.types string @@ -2567,38 +2528,39 @@ STAGE PLANS: /merge_src_part/ds=2008-04-09 [merge_src_part] Needs Tagging: false Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2 - columns.types struct:struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.merge_src_part2 + partition_columns ds + partition_columns.types string + serialization.ddl struct merge_src_part2 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_src_part2 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Stage: Stage-7 Conditional Operator @@ -4907,12 +4869,11 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) - null sort order: a - sort order: + + null sort order: aa + sort order: ++ Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) auto parallelism: false Execution mode: vectorized Path -> Alias: @@ -4934,7 +4895,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.merge_src_part - numFiles 2 + numFiles 1 numRows 1000 partition_columns ds partition_columns.types string @@ -4982,7 +4943,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.merge_src_part - numFiles 2 + numFiles 1 numRows 1000 partition_columns ds partition_columns.types string @@ -5020,13 +4981,13 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 #### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED NumFilesPerFileSink: 1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### @@ -5052,42 +5013,6 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, ds - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - properties: - columns _col0,_col1,_col2 - columns.types struct:struct:string - escape.delim \ - hive.serialization.extend.additional.nesting.levels true - serialization.escape.crlf true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/merge4.q.out b/ql/src/test/results/clientpositive/merge4.q.out index 78561ceeb41651b372ee51f0040241d93fcbb336..fcc557a5ae2b5360a8b063da5680205c10321e8c 100644 --- a/ql/src/test/results/clientpositive/merge4.q.out +++ b/ql/src/test/results/clientpositive/merge4.q.out @@ -41,48 +41,25 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2010-08-15' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out b/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out index 7d12b58c51ef17b34a15d96b8d22749da685f998..6e75c6e0624dfc340568f4dc0d4c567d145ad54f 100644 --- a/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out +++ b/ql/src/test/results/clientpositive/merge_dynamic_partition.q.out @@ -75,48 +75,24 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 28766 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 28766 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 49 Data size: 28766 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n1 Stage: Stage-0 Move Operator @@ -672,10 +648,10 @@ outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat columns:struct columns { string key, string value} partitioned:true partitionColumns:struct partition_columns { string ds, string hr} -totalNumberFiles:4 +totalNumberFiles:1 totalFileSize:5812 -maxFileSize:1612 -minFileSize:1358 +maxFileSize:5812 +minFileSize:5812 #### A masked pattern was here #### PREHOOK: query: explain @@ -1387,48 +1363,24 @@ STAGE PLANS: expressions: key (type: string), value (type: string), '2008-04-08' (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 28766 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 49 Data size: 28766 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 49 Data size: 28766 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 99 Data size: 58120 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n1 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out b/ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out index 24502841c35ee0662f9d0d09918ede402874e9d7..a98bfbffecd42d62478831d905469ad7195d34b6 100644 --- a/ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out +++ b/ql/src/test/results/clientpositive/merge_dynamic_partition2.q.out @@ -99,48 +99,24 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n0 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 148 Data size: 86781 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 148 Data size: 86781 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 148 Data size: 86781 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n0 Stage: Stage-7 Conditional Operator @@ -230,9 +206,9 @@ outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat columns:struct columns { string key, string value} partitioned:true partitionColumns:struct partition_columns { string ds, string hr} -totalNumberFiles:3 +totalNumberFiles:2 totalFileSize:17415 -maxFileSize:5901 -minFileSize:5702 +maxFileSize:11603 +minFileSize:5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out b/ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out index d07ad0ad4bc666cc7b21e97a85a969fc32ae44ba..cbc8afd10450ba63c530936104b24931a1da5dd7 100644 --- a/ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out +++ b/ql/src/test/results/clientpositive/merge_dynamic_partition3.q.out @@ -163,48 +163,24 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 594 Data size: 348300 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 594 Data size: 348300 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.merge_dynamic_part_n2 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 594 Data size: 348300 Basic stats: PARTIAL Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 594 Data size: 348300 Basic stats: PARTIAL Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 594 Data size: 348300 Basic stats: PARTIAL Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 297 Data size: 174150 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 594 Data size: 348300 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.merge_dynamic_part_n2 Stage: Stage-7 Conditional Operator @@ -324,9 +300,9 @@ outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat columns:struct columns { string key, string value} partitioned:true partitionColumns:struct partition_columns { string ds, string hr} -totalNumberFiles:6 +totalNumberFiles:4 totalFileSize:34830 -maxFileSize:5812 -minFileSize:5791 +maxFileSize:11603 +minFileSize:5812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out b/ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out index 545ef90c7f20ad8b0340b8ff6c9ba7998096bb66..8a4b96f38c076c3758921ee1efe2491b6de09fa7 100644 --- a/ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out +++ b/ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out @@ -160,48 +160,24 @@ STAGE PLANS: expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 2.0D) = 0.0D), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.merge_dynamic_part_n3 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part_n3 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out b/ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out index 6d994d75f61e337546806b9072b07d9b6e391478..db782cbd07c8e47b4db80adc562d20495c1d849a 100644 --- a/ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out +++ b/ql/src/test/results/clientpositive/merge_dynamic_partition5.q.out @@ -136,48 +136,24 @@ STAGE PLANS: expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0D) = 0.0D), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 618 Data size: 5934 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 618 Data size: 5934 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.merge_dynamic_part - Select Operator - expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 618 Data size: 5934 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 618 Data size: 5934 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 618 Data size: 5934 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 309 Data size: 2967 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 309 Data size: 2967 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 309 Data size: 2967 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 618 Data size: 5934 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/orc_int_type_promotion.q.out b/ql/src/test/results/clientpositive/orc_int_type_promotion.q.out index 7a1fb44c779bcb288fdb6f2b6c9e17cc8a290bfe..b6fb7e2dc299ed81bf58459328c0c6e3f6eb8ab9 100644 --- a/ql/src/test/results/clientpositive/orc_int_type_promotion.q.out +++ b/ql/src/test/results/clientpositive/orc_int_type_promotion.q.out @@ -262,15 +262,15 @@ POSTHOOK: Input: default@src_part_orc@ds=2008-04-08 POSTHOOK: Input: default@src_part_orc@ds=2008-04-09 #### A masked pattern was here #### 238 val_238 2008-04-08 -86 val_86 2008-04-08 -311 val_311 2008-04-08 -27 val_27 2008-04-08 -165 val_165 2008-04-08 -409 val_409 2008-04-08 -255 val_255 2008-04-08 -278 val_278 2008-04-08 -98 val_98 2008-04-08 -484 val_484 2008-04-08 +97 val_97 2008-04-08 +200 val_200 2008-04-08 +400 val_400 2008-04-08 +403 val_403 2008-04-08 +169 val_169 2008-04-08 +90 val_90 2008-04-08 +126 val_126 2008-04-08 +222 val_222 2008-04-08 +477 val_477 2008-04-08 PREHOOK: query: alter table src_part_orc change key key bigint PREHOOK: type: ALTERTABLE_RENAMECOL PREHOOK: Input: default@src_part_orc @@ -292,12 +292,12 @@ POSTHOOK: Input: default@src_part_orc@ds=2008-04-08 POSTHOOK: Input: default@src_part_orc@ds=2008-04-09 #### A masked pattern was here #### 238 val_238 2008-04-08 -86 val_86 2008-04-08 -311 val_311 2008-04-08 -27 val_27 2008-04-08 -165 val_165 2008-04-08 -409 val_409 2008-04-08 -255 val_255 2008-04-08 -278 val_278 2008-04-08 -98 val_98 2008-04-08 -484 val_484 2008-04-08 +97 val_97 2008-04-08 +200 val_200 2008-04-08 +400 val_400 2008-04-08 +403 val_403 2008-04-08 +169 val_169 2008-04-08 +90 val_90 2008-04-08 +126 val_126 2008-04-08 +222 val_222 2008-04-08 +477 val_477 2008-04-08 diff --git a/ql/src/test/results/clientpositive/orc_merge1.q.out b/ql/src/test/results/clientpositive/orc_merge1.q.out index 681dd573e87d1127710b032ab4b51fa16da8e0b8..83ad4c13c1f5d975cd6fc8c6d36ee74edcb4b2d6 100644 --- a/ql/src/test/results/clientpositive/orc_merge1.q.out +++ b/ql/src/test/results/clientpositive/orc_merge1.q.out @@ -69,48 +69,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n1 Stage: Stage-0 Move Operator @@ -150,7 +126,7 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 2 items +Found 1 items #### A masked pattern was here #### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part) @@ -186,48 +162,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n1 Stage: Stage-7 Conditional Operator @@ -342,48 +294,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n1 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/orc_merge10.q.out b/ql/src/test/results/clientpositive/orc_merge10.q.out index f9dda1176868f6ce42f73966989aff05546ba75d..0184fa9c1bf97c5139b2e12cf94c85276d616fcf 100644 --- a/ql/src/test/results/clientpositive/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/orc_merge10.q.out @@ -69,48 +69,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 Stage: Stage-0 Move Operator @@ -150,7 +126,7 @@ POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)sr POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 2 items +Found 1 items #### A masked pattern was here #### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) @@ -186,48 +162,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b Stage: Stage-7 Conditional Operator @@ -342,48 +294,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c Stage: Stage-7 Conditional Operator @@ -640,61 +568,41 @@ Type: struct Stripe Statistics: Stripe 1: - Column 0: count: 152 hasNull: false - Column 1: count: 152 hasNull: false bytesOnDisk: 309 min: 0 max: 497 sum: 38034 - Column 2: count: 152 hasNull: false bytesOnDisk: 679 min: val_0 max: val_97 sum: 1034 - Stripe 2: - Column 0: count: 90 hasNull: false - Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736 - Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612 + Column 0: count: 242 hasNull: false + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 File Statistics: Column 0: count: 242 hasNull: false - Column 1: count: 242 hasNull: false bytesOnDisk: 494 min: 0 max: 497 sum: 60770 - Column 2: count: 242 hasNull: false bytesOnDisk: 1107 min: val_0 max: val_97 sum: 1646 + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 Stripes: - Stripe: offset: 3 data: 988 rows: 152 tail: 72 index: 77 + Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77 Stream: column 0 section ROW_INDEX start: 3 length 12 Stream: column 1 section ROW_INDEX start: 15 length 28 Stream: column 2 section ROW_INDEX start: 43 length 37 - Stream: column 1 section DATA start: 80 length 309 - Stream: column 2 section DATA start: 389 length 157 - Stream: column 2 section LENGTH start: 546 length 60 - Stream: column 2 section DICTIONARY_DATA start: 606 length 462 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DICTIONARY_V2[114] - Row group indices for column 0: - Entry 0: count: 152 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 152 hasNull: false min: 0 max: 497 sum: 38034 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 152 hasNull: false min: val_0 max: val_97 sum: 1034 positions: 0,0,0 - Stripe: offset: 1140 data: 613 rows: 90 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 1140 length 11 - Stream: column 1 section ROW_INDEX start: 1151 length 27 - Stream: column 2 section ROW_INDEX start: 1178 length 38 - Stream: column 1 section DATA start: 1216 length 185 - Stream: column 2 section DATA start: 1401 length 377 - Stream: column 2 section LENGTH start: 1778 length 51 + Stream: column 1 section DATA start: 80 length 489 + Stream: column 2 section DATA start: 569 length 247 + Stream: column 2 section LENGTH start: 816 length 71 + Stream: column 2 section DICTIONARY_DATA start: 887 length 592 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 + Encoding column 2: DICTIONARY_V2[153] Row group indices for column 0: - Entry 0: count: 90 hasNull: false positions: + Entry 0: count: 242 hasNull: false positions: Row group indices for column 1: - Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0 + Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0 Row group indices for column 2: - Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0 + Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 2155 bytes +File length: 1754 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ -- END ORC FILE DUMP -- -172 val_172 1 0 +103 val_103 1 0 PREHOOK: query: select * from orcfile_merge1c where ds='1' and part='0' limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@orcfile_merge1c @@ -710,61 +618,41 @@ Type: struct Stripe Statistics: Stripe 1: - Column 0: count: 152 hasNull: false - Column 1: count: 152 hasNull: false bytesOnDisk: 309 min: 0 max: 497 sum: 38034 - Column 2: count: 152 hasNull: false bytesOnDisk: 679 min: val_0 max: val_97 sum: 1034 - Stripe 2: - Column 0: count: 90 hasNull: false - Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736 - Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612 + Column 0: count: 242 hasNull: false + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 File Statistics: Column 0: count: 242 hasNull: false - Column 1: count: 242 hasNull: false bytesOnDisk: 494 min: 0 max: 497 sum: 60770 - Column 2: count: 242 hasNull: false bytesOnDisk: 1107 min: val_0 max: val_97 sum: 1646 + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 Stripes: - Stripe: offset: 3 data: 988 rows: 152 tail: 72 index: 77 + Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77 Stream: column 0 section ROW_INDEX start: 3 length 12 Stream: column 1 section ROW_INDEX start: 15 length 28 Stream: column 2 section ROW_INDEX start: 43 length 37 - Stream: column 1 section DATA start: 80 length 309 - Stream: column 2 section DATA start: 389 length 157 - Stream: column 2 section LENGTH start: 546 length 60 - Stream: column 2 section DICTIONARY_DATA start: 606 length 462 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DICTIONARY_V2[114] - Row group indices for column 0: - Entry 0: count: 152 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 152 hasNull: false min: 0 max: 497 sum: 38034 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 152 hasNull: false min: val_0 max: val_97 sum: 1034 positions: 0,0,0 - Stripe: offset: 1140 data: 613 rows: 90 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 1140 length 11 - Stream: column 1 section ROW_INDEX start: 1151 length 27 - Stream: column 2 section ROW_INDEX start: 1178 length 38 - Stream: column 1 section DATA start: 1216 length 185 - Stream: column 2 section DATA start: 1401 length 377 - Stream: column 2 section LENGTH start: 1778 length 51 + Stream: column 1 section DATA start: 80 length 489 + Stream: column 2 section DATA start: 569 length 247 + Stream: column 2 section LENGTH start: 816 length 71 + Stream: column 2 section DICTIONARY_DATA start: 887 length 592 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 + Encoding column 2: DICTIONARY_V2[153] Row group indices for column 0: - Entry 0: count: 90 hasNull: false positions: + Entry 0: count: 242 hasNull: false positions: Row group indices for column 1: - Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0 + Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0 Row group indices for column 2: - Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0 + Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 2155 bytes +File length: 1754 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ -- END ORC FILE DUMP -- -172 val_172 1 0 +103 val_103 1 0 PREHOOK: query: DROP TABLE orcfile_merge1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@orcfile_merge1 diff --git a/ql/src/test/results/clientpositive/orc_merge2.q.out b/ql/src/test/results/clientpositive/orc_merge2.q.out index ecb528f801e086dc1969122fc90230b7b711f7ee..b91d9812fcca92e34c5ddb48ec3772f2b6298ee8 100644 --- a/ql/src/test/results/clientpositive/orc_merge2.q.out +++ b/ql/src/test/results/clientpositive/orc_merge2.q.out @@ -29,13 +29,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -48,57 +43,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge2a_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) - outputColumnNames: key, value, one, two, three - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: one (type: string), two (type: string), three (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col3 (type: struct), _col4 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a_n0 Stage: Stage-0 Move Operator @@ -122,26 +84,6 @@ STAGE PLANS: Column Types: int, string Table: default.orcfile_merge2a_n0 - Stage: Stage-3 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-5 - Merge File Operator - Map Operator Tree: - ORC File Merge Operator - merge level: stripe - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge2a_n0 PARTITION (one='1', two, three) SELECT key, value, PMOD(HASH(key), 10) as two, PMOD(HASH(value), 10) as three diff --git a/ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out b/ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out index ab46126fa78653c9834bbc531968b9a172cd68b3..d26cf05388da2d396d27b82516dd95422b03efff 100644 --- a/ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out +++ b/ql/src/test/results/clientpositive/orc_merge_diff_fs.q.out @@ -69,48 +69,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n0 Stage: Stage-0 Move Operator @@ -150,7 +126,7 @@ POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 2 items +Found 1 items #### A masked pattern was here #### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n0 PARTITION (ds='1', part) @@ -186,48 +162,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n0 Stage: Stage-7 Conditional Operator @@ -342,48 +294,24 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n0 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out index a0a94b58d1a25ee8f4463c0f6494fd744c2265ce..1a4ff2f3fcccc20b20fa2e7411f0c2cdb7da749b 100644 --- a/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out +++ b/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out @@ -45,48 +45,24 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double) - outputColumnNames: userid, string1, subtype, decimal1, ts, st - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll') - keys: st (type: double) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4) - keys: KEY._col0 (type: double) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out index 6510a3930bdb2a9d02165999ec0f6f2817edc318..24cf07b9a3e8e22f82a4fbd910020d6160cbef04 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out @@ -442,20 +442,12 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 +169 val_169 NULL day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 -374 val_374 val_374 day1 1 -374 val_374 val_374 day1 1 +374 val_374 NULL day1 1 PREHOOK: query: drop table bucket_big_n17 PREHOOK: type: DROPTABLE PREHOOK: Input: default@bucket_big_n17 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark index 5b759d217cd2c488236bd70fdc2b4ad4cacaceda..8c2244048869b79ed59a3bfdf6022ec370c27f5f 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_16.q.out_spark @@ -442,20 +442,12 @@ POSTHOOK: Input: default@bucket_small_n17@pri=2 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 103 val_103 val_103 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 +169 val_169 NULL day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 172 val_172 val_172 day1 1 -374 val_374 val_374 day1 1 -374 val_374 val_374 day1 1 +374 val_374 NULL day1 1 PREHOOK: query: drop table bucket_big_n17 PREHOOK: type: DROPTABLE PREHOOK: Input: default@bucket_big_n17 diff --git a/ql/src/test/results/clientpositive/spark/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/spark/dynpart_sort_optimization.q.out new file mode 100644 index 0000000000000000000000000000000000000000..ddc5106bf572877b45f061542123e52912931b62 --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/dynpart_sort_optimization.q.out @@ -0,0 +1,3504 @@ +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part +POSTHOOK: query: create table over1k_part( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part +PREHOOK: query: create table over1k_part_limit like over1k_part +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_limit +POSTHOOK: query: create table over1k_part_limit like over1k_part +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_limit +PREHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: create table over1k_part_buck( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck +PREHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: create table over1k_part_buck_sort( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 4 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert overwrite table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert overwrite table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert overwrite table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: explain insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_limit + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: explain insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: explain insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: insert into table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_limit@ds=foo +POSTHOOK: query: insert into table over1k_part_limit partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_limit PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck +POSTHOOK: query: insert into table over1k_part_buck partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck@t=27 +POSTHOOK: Output: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort +POSTHOOK: query: insert into table over1k_part_buck_sort partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part +POSTHOOK: query: desc formatted over1k_part partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 14 + rawDataSize 362 + totalSize 376 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_limit +POSTHOOK: query: desc formatted over1k_part_limit partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_limit +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_limit +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck +POSTHOOK: query: desc formatted over1k_part_buck partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 32 + rawDataSize 830 + totalSize 862 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: query: desc formatted over1k_part_buck_sort partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 8 + numRows 6 + rawDataSize 156 + totalSize 162 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 4 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select count(*) from over1k_part +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part +PREHOOK: Input: default@over1k_part@ds=foo/t=27 +PREHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part +POSTHOOK: Input: default@over1k_part@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_limit +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_limit +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +PREHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_limit +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_limit +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part_limit@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +20 +PREHOOK: query: select count(*) from over1k_part_buck +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck +PREHOOK: Input: default@over1k_part_buck@t=27 +PREHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck +POSTHOOK: Input: default@over1k_part_buck@t=27 +POSTHOOK: Input: default@over1k_part_buck@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: select count(*) from over1k_part_buck_sort +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort +PREHOOK: Input: default@over1k_part_buck_sort@t=27 +PREHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort +POSTHOOK: Input: default@over1k_part_buck_sort@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +38 +PREHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part2 +POSTHOOK: query: create table over1k_part2( + si smallint, + i int, + b bigint, + f float) + partitioned by (ds string, t tinyint) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part2 +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: tinyint), VALUE._col1 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 = 27Y) or _col0 is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: t (type: tinyint), si (type: smallint), i (type: int), b (type: bigint), f (type: float) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + sort order: +++++ + Map-reduce partition columns: _col0 (type: tinyint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part2@ds=foo +POSTHOOK: query: insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Output: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part2 PARTITION(ds=foo,t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, 27] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part2 +POSTHOOK: query: desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +ds string +t tinyint + +# Detailed Partition Information +Partition Value: [foo, __HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +405 65536 4294967508 82.24 foo 27 +457 65570 4294967464 81.58 foo 27 +256 65599 4294967383 89.55 foo 27 +335 65617 4294967381 64.87 foo 27 +261 65619 4294967401 88.78 foo 27 +278 65622 4294967516 25.67 foo 27 +482 65624 4294967313 78.98 foo 27 +503 65628 4294967371 95.07 foo 27 +335 65636 4294967505 37.14 foo 27 +367 65675 4294967518 12.32 foo 27 +340 65677 4294967461 98.96 foo 27 +490 65680 4294967347 57.46 foo 27 +287 65708 4294967542 83.33 foo 27 +329 65778 4294967451 6.63 foo 27 +401 65779 4294967402 97.39 foo 27 +262 65787 4294967371 57.35 foo 27 +409 65536 4294967490 46.97 foo NULL +374 65560 4294967516 65.43 foo NULL +473 65720 4294967324 80.74 foo NULL +PREHOOK: query: select count(*) from over1k_part2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part2 +PREHOOK: Input: default@over1k_part2@ds=foo/t=27 +PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part2 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=27 +POSTHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: create table over1k_part_buck_sort2( + si smallint, + i int, + b bigint, + f float) + partitioned by (t tinyint) + clustered by (si) + sorted by (f) into 1 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part_buck_sort2 +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: float) + sort order: + + Map-reduce partition columns: _col0 (type: smallint) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-0 + Move Operator + tables: + partition: + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part_buck_sort2 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part_buck_sort2 +POSTHOOK: query: insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Output: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=27).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).i SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1k_part_buck_sort2 PARTITION(t=__HIVE_DEFAULT_PARTITION__).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t=27) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [27] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 16 + rawDataSize 415 + totalSize 431 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: query: desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@over1k_part_buck_sort2 +# col_name data_type comment +si smallint +i int +b bigint +f float + +# Partition Information +# col_name data_type comment +t tinyint + +# Detailed Partition Information +Partition Value: [__HIVE_DEFAULT_PARTITION__] +Database: default +Table: over1k_part_buck_sort2 +#### A masked pattern was here #### +Partition Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 3 + rawDataSize 78 + totalSize 81 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: 1 +Bucket Columns: [si] +Sort Columns: [Order(col:f, order:1)] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select * from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +329 65778 4294967451 6.63 27 +367 65675 4294967518 12.32 27 +278 65622 4294967516 25.67 27 +335 65636 4294967505 37.14 27 +262 65787 4294967371 57.35 27 +490 65680 4294967347 57.46 27 +335 65617 4294967381 64.87 27 +482 65624 4294967313 78.98 27 +457 65570 4294967464 81.58 27 +405 65536 4294967508 82.24 27 +287 65708 4294967542 83.33 27 +261 65619 4294967401 88.78 27 +256 65599 4294967383 89.55 27 +503 65628 4294967371 95.07 27 +401 65779 4294967402 97.39 27 +340 65677 4294967461 98.96 27 +409 65536 4294967490 46.97 NULL +374 65560 4294967516 65.43 NULL +473 65720 4294967324 80.74 NULL +PREHOOK: query: select count(*) from over1k_part_buck_sort2 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part_buck_sort2 +PREHOOK: Input: default@over1k_part_buck_sort2@t=27 +PREHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from over1k_part_buck_sort2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part_buck_sort2 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=27 +POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__ +#### A masked pattern was here #### +19 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (s = 'foo') (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (t = 27Y) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (i = 100) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), t (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), i (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + sort order: +++ + Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) + value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: ((i = 100) and (t = 27Y) and (s = 'foo')) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((i = 100) and (s = 'foo') and (t = 27Y)) (type: boolean) + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27Y (type: tinyint), 100 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 1066360 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-0 + Move Operator + tables: + partition: + i + s + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part3 + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_part3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_part3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: drop table over1k_part3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: create table over1k_part3( + si smallint, + b bigint, + f float) + partitioned by (s string, t tinyint, i int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_part3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k_n3)over1k_n3.FieldSchema(name:si, type:smallint, comment:null), ] +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part3 +POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +PREHOOK: query: select sum(hash(*)) from over1k_part3 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_part3 +PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +POSTHOOK: query: select sum(hash(*)) from over1k_part3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_part3 +POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636 +POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619 +POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617 +POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628 +POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787 +POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778 +POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779 +POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624 +POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536 +POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622 +POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708 +POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677 +POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675 +POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570 +POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 +POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 +#### A masked pattern was here #### +17814641134 +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: analyze table over1k_n3 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +POSTHOOK: query: analyze table over1k_n3 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Stats Work + Basic Stats Work: + +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out index 090e9cb3acfc2ecdb160049d3029460d99e71ced..3ca922d3c6cbf4117fab2da658faa6e6d4af18de 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part1.q.out @@ -73,9 +73,12 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark + Edges: + Reducer 2 <- Map 4 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: - Map 1 + Map 4 Map Operator Tree: TableScan alias: srcpart @@ -87,14 +90,17 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1_n0 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized + Map 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (ds > '2008-04-08') (type: boolean) Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE @@ -102,15 +108,42 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part2_n0 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part1_n0 + Reducer 3 Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part2_n0 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out index 1885f9cd009221d23d42b81c08b8197a43392136..5fd3a04f0809984e454f8cf056fe2796ce754e25 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part10.q.out @@ -56,6 +56,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -68,15 +70,27 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part10 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part10 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out index 4e90e95010861985d6901b7d75a50794c8a23ad1..625d60c9d19fb23ca8dc855be9f0b59144a0db08 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part3.q.out @@ -58,6 +58,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -70,15 +72,27 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part3 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part3 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out index 2a0eddb54a4045264bcf7905fc49c6c0d7ffc45a..811becc694154a13af179f6b3b834f3a51f16a1e 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part4.q.out @@ -68,6 +68,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -80,15 +82,27 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part4 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part4 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out index 33815e0a12779e6228374a0d5fe4b5600022ca1f..8e9f81302091beb40d2611370d4f379c482abee3 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part5.q.out @@ -37,6 +37,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -48,15 +50,27 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part5 + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: string) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part5 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out index b59189a5d17b5af8f877557644ec0b685eda70b7..aebf4382cd78b02d9b7bab7285254431f04e29c0 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part8.q.out @@ -65,9 +65,12 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-2 Spark + Edges: + Reducer 2 <- Map 4 (PARTITION-LEVEL SORT, 2) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: - Map 1 + Map 4 Map Operator Tree: TableScan alias: srcpart @@ -81,35 +84,225 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + null sort order: aa + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false + Execution mode: vectorized + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + hr 11 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart +#### A masked pattern was here #### + Partition + base file name: hr=12 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-09 + hr 12 + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + bucketing_version 2 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string #### A masked pattern was here #### - name default.nzhang_part8_n0 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8_n0 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8_n0 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [srcpart] + /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-09/hr=11 [srcpart] + /srcpart/ds=2008-04-09/hr=12 [srcpart] + Map 5 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + GatherStats: false Filter Operator isSamplingPred: false predicate: (ds > '2008-04-08') (type: boolean) @@ -118,36 +311,14 @@ STAGE PLANS: expressions: key (type: string), value (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 2 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Static Partition Specification: ds=2008-12-31/ - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count -1 - column.name.delimiter , - columns key,value - columns.comments 'default','default' - columns.types string:string -#### A masked pattern was here #### - name default.nzhang_part8_n0 - partition_columns ds/hr - partition_columns.types string:string - serialization.ddl struct nzhang_part8_n0 { string key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -#### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part8_n0 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + Reduce Output Operator + key expressions: _col2 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col2 (type: string) + tag: -1 + value expressions: _col0 (type: string), _col1 (type: string) + auto parallelism: false Execution mode: vectorized Path -> Alias: #### A masked pattern was here #### @@ -353,6 +524,81 @@ STAGE PLANS: /srcpart/ds=2008-04-08/hr=12 [srcpart] /srcpart/ds=2008-04-09/hr=11 [srcpart] /srcpart/ds=2008-04-09/hr=12 [srcpart] + Reducer 2 + Execution mode: vectorized + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.nzhang_part8_n0 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8_n0 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8_n0 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Reducer 3 + Execution mode: vectorized + Needs Tagging: false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 2 +#### A masked pattern was here #### + Dp Sort State: PARTITION_SORTED + NumFilesPerFileSink: 1 + Static Partition Specification: ds=2008-12-31/ + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.nzhang_part8_n0 + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct nzhang_part8_n0 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part8_n0 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out index f93a255386d5b29d714622690ec77543f8053d61..b62acc40905ffefef4a78c4d27a0cccbbec7bd32 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part9.q.out @@ -56,6 +56,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -68,15 +70,27 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part9 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part9 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/orc_merge1.q.out b/ql/src/test/results/clientpositive/spark/orc_merge1.q.out index 977c4cbfc139be8c61a05e70159275469b157169..49f9124f288ac520000eef3556774ce32059ae79 100644 --- a/ql/src/test/results/clientpositive/spark/orc_merge1.q.out +++ b/ql/src/test/results/clientpositive/spark/orc_merge1.q.out @@ -61,6 +61,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -72,14 +74,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n1 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n1 Stage: Stage-0 Move Operator @@ -115,9 +128,8 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 2 items --rw-r--r-- 3 ### USER ### ### GROUP ### 752 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 1056 ### HDFS DATE ### hdfs://### HDFS PATH ### +Found 1 items +-rw-r--r-- 3 ### USER ### ### GROUP ### 1367 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -144,6 +156,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -155,14 +169,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n1 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n1 Stage: Stage-7 Conditional Operator @@ -244,7 +269,7 @@ POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-r--r-- 3 ### USER ### ### GROUP ### 1360 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-r--r-- 3 ### USER ### ### GROUP ### 1367 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -271,6 +296,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -282,14 +309,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n1 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n1 Stage: Stage-7 Conditional Operator @@ -363,7 +401,7 @@ POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-r--r-- 3 ### USER ### ### GROUP ### 1669 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-r--r-- 3 ### USER ### ### GROUP ### 1367 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1_n1 WHERE ds='1' diff --git a/ql/src/test/results/clientpositive/spark/orc_merge2.q.out b/ql/src/test/results/clientpositive/spark/orc_merge2.q.out index ae9750e1df26a2847a26163df91e0ad99e144809..6d571b1732c939bcfdb70436fd977d9bbb339873 100644 --- a/ql/src/test/results/clientpositive/spark/orc_merge2.q.out +++ b/ql/src/test/results/clientpositive/spark/orc_merge2.q.out @@ -40,6 +40,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -51,14 +53,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge2a_n0 + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a_n0 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/spark/orc_merge7.q.out b/ql/src/test/results/clientpositive/spark/orc_merge7.q.out index 8ce15470045b1f0de619a025f26a6e4b2283cc37..192f8c46c8bca51ce2c346a012df92ee2f73697c 100644 --- a/ql/src/test/results/clientpositive/spark/orc_merge7.q.out +++ b/ql/src/test/results/clientpositive/spark/orc_merge7.q.out @@ -37,6 +37,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -48,14 +50,25 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a_n0 + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a_n0 Stage: Stage-0 Move Operator @@ -208,6 +221,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -219,14 +234,25 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a_n0 + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a_n0 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out b/ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out index 6bca5720a7020cd4b0a44a034d85e6f781982e3a..357cbfa285038a18712129b37668f2be989b81ab 100644 --- a/ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out +++ b/ql/src/test/results/clientpositive/spark/orc_merge_diff_fs.q.out @@ -61,6 +61,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -72,14 +74,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n0 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n0 Stage: Stage-0 Move Operator @@ -115,7 +128,7 @@ POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 2 items +Found 1 items #### A masked pattern was here #### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n0 PARTITION (ds='1', part) @@ -143,6 +156,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -154,14 +169,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n0 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n0 Stage: Stage-7 Conditional Operator @@ -270,6 +296,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -281,14 +309,25 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n0 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n0 Stage: Stage-7 Conditional Operator diff --git a/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out index 838888d8127fbe77e3ea86fe8dd3e904de0f1f1f..2330d9e27288e9ba89d8d4fa22ff587cc37f1f08 100644 --- a/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out +++ b/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out @@ -37,6 +37,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4) #### A masked pattern was here #### Vertices: Map 1 @@ -48,14 +50,25 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 22980 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/stats2.q.out b/ql/src/test/results/clientpositive/spark/stats2.q.out index 55d2cb35361334aa5f717b99d1987c1139782d13..30339caeb2cff5cc96101d8cbf5f3ed8b5b01667 100644 --- a/ql/src/test/results/clientpositive/spark/stats2.q.out +++ b/ql/src/test/results/clientpositive/spark/stats2.q.out @@ -30,6 +30,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -42,15 +44,27 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.analyze_t1 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_t1 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/union_remove_17.q.out b/ql/src/test/results/clientpositive/spark/union_remove_17.q.out index ab250fe229d5bedc6c9b3cee125b90ebd091f4b4..1249138ed828680b0f705e612d4265164428cdfd 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_17.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_17.q.out @@ -50,6 +50,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -65,16 +67,13 @@ STAGE PLANS: expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n4 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: bigint) Execution mode: vectorized - Map 2 + Map 3 Map Operator Tree: TableScan alias: inputtbl1_n3 @@ -87,15 +86,27 @@ STAGE PLANS: expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n4 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: bigint) Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n4 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out index d63819f5614c903dc1c00cf7c454e617f4204127..cbf37d74860297f6a2a3480bc23518f00fe23d4a 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_25.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_25.q.out @@ -429,7 +429,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) - Reducer 4 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) #### A masked pattern was here #### Vertices: Map 1 @@ -465,36 +465,26 @@ STAGE PLANS: expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl3_n3 - Reducer 4 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: bigint) + Reducer 3 Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 1000 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), '2008-04-08' (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.outputtbl3_n3 + expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.outputtbl3_n3 Stage: Stage-0 Move Operator @@ -567,7 +557,7 @@ Database: default Table: outputtbl3_n3 #### A masked pattern was here #### Partition Parameters: - numFiles 2 + numFiles 1 totalSize 6812 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/stats2.q.out b/ql/src/test/results/clientpositive/stats2.q.out index af205c1fb042693304e4e88bf072ba7e964e2d24..43c12387331b87763a4a44d7caa273891efaf9d0 100644 --- a/ql/src/test/results/clientpositive/stats2.q.out +++ b/ql/src/test/results/clientpositive/stats2.q.out @@ -39,15 +39,25 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.analyze_t1 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.analyze_t1 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/stats4.q.out b/ql/src/test/results/clientpositive/stats4.q.out index e1ca68fca2f855991aa3cad2bc2359a705461193..b1edea1535d1711b49cbc351aaca115390688b67 100644 --- a/ql/src/test/results/clientpositive/stats4.q.out +++ b/ql/src/test/results/clientpositive/stats4.q.out @@ -56,16 +56,11 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-2 is a root stage - Stage-8 depends on stages: Stage-2 , consists of Stage-5, Stage-4, Stage-6 - Stage-5 - Stage-0 depends on stages: Stage-5, Stage-4, Stage-7 - Stage-3 depends on stages: Stage-0, Stage-10 - Stage-4 - Stage-6 - Stage-7 depends on stages: Stage-6 - Stage-1 depends on stages: Stage-2 - Stage-9 depends on stages: Stage-1, Stage-10 - Stage-10 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + Stage-4 depends on stages: Stage-2 + Stage-1 depends on stages: Stage-4 + Stage-5 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-2 @@ -81,30 +76,11 @@ STAGE PLANS: expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) - outputColumnNames: key, value, ds, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: string) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) Filter Operator predicate: (ds > '2008-04-08') (type: boolean) Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE @@ -114,55 +90,24 @@ STAGE PLANS: Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part2 - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: key, value, hr - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: '2008-12-31' (type: string), hr (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-8 - Conditional Operator - - Stage: Stage-5 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part1 Stage: Stage-0 Move Operator @@ -189,31 +134,25 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1 - - Stage: Stage-6 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part1 - - Stage: Stage-7 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part2 Stage: Stage-1 Move Operator @@ -228,7 +167,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.nzhang_part2 - Stage: Stage-9 + Stage: Stage-5 Stats Work Basic Stats Work: Column Stats Desc: @@ -236,36 +175,6 @@ STAGE PLANS: Column Types: string, string Table: default.nzhang_part2 - Stage: Stage-10 - Map Reduce - Map Operator Tree: - TableScan - Reduce Output Operator - key expressions: '2008-12-31' (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: '2008-12-31' (type: string), _col1 (type: string) - Statistics: Num rows: 666 Data size: 7075 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct) - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: '2008-12-31' (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), '2008-12-31' (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - PREHOOK: query: from srcpart insert overwrite table nzhang_part1 partition (ds, hr) select key, value, ds, hr where ds <= '2008-04-08' insert overwrite table nzhang_part2 partition(ds='2008-12-31', hr) select key, value, hr where ds > '2008-04-08' diff --git a/ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out b/ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out index 7825cb25f17b6b2b92053d536920789fb21acfd4..431ff3ff71194f96f99c383870d0bd7058111651 100644 --- a/ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out +++ b/ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out @@ -15,13 +15,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5 - Stage-4 - Stage-0 depends on stages: Stage-4, Stage-3, Stage-6 + Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 - Stage-3 - Stage-5 - Stage-6 depends on stages: Stage-5 STAGE PLANS: Stage: Stage-1 @@ -38,57 +33,25 @@ STAGE PLANS: expressions: 'no_such_value' (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n7 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, part - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(key, 'hll') - keys: part (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct) + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: string) + Execution mode: vectorized Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-7 - Conditional Operator - - Stage: Stage-4 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable_n7 Stage: Stage-0 Move Operator @@ -110,36 +73,6 @@ STAGE PLANS: Column Types: string Table: default.tmptable_n7 - Stage: Stage-3 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n7 - - Stage: Stage-5 - Map Reduce - Map Operator Tree: - TableScan - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmptable_n7 - - Stage: Stage-6 - Move Operator - files: - hdfs directory: true -#### A masked pattern was here #### - PREHOOK: query: insert overwrite table tmptable_n7 partition (part) select key, value from src where key = 'no_such_value' PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git a/ql/src/test/results/clientpositive/union_remove_17.q.out b/ql/src/test/results/clientpositive/union_remove_17.q.out index 9401187b5678bd7238db548829e7e52b9287caaf..d01329c6448bc699c79a23e6db3f3260d647cc94 100644 --- a/ql/src/test/results/clientpositive/union_remove_17.q.out +++ b/ql/src/test/results/clientpositive/union_remove_17.q.out @@ -64,14 +64,11 @@ STAGE PLANS: expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n4 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: bigint) TableScan alias: inputtbl1_n3 Statistics: Num rows: 1 Data size: 300 Basic stats: COMPLETE Column stats: NONE @@ -85,14 +82,24 @@ STAGE PLANS: expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat - output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat - serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - name: default.outputtbl1_n4 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: string), _col1 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 2 Data size: 600 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.outputtbl1_n4 Stage: Stage-0 Move Operator