diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b600518511..a1412f42f1 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2226,6 +2226,16 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "When enabled dynamic partitioning column will be globally sorted.\n" + "This way we can keep only one record writer open for each partition value\n" + "in the reducer thereby reducing the memory pressure on reducers."), + HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD("hive.optimize.sort.dynamic.partition.threshold", 0, + "When enabled dynamic partitioning column will be globally sorted.\n" + + "This way we can keep only one record writer open for each partition value\n" + + "in the reducer thereby reducing the memory pressure on reducers.\n" + + "This config has following possible values: \n" + + "\t-1 - This completely disables the optimization. \n" + + "\t1 - This always enable the optimization. \n" + + "\t0 - This makes the optimization a cost based decision. \n" + + "Setting it to any other positive integer will make Hive use this as threshold for number of writers."), + HIVESAMPLINGFORORDERBY("hive.optimize.sampling.orderby", false, "Uses sampling on order-by clause for parallel execution."), HIVESAMPLINGNUMBERFORORDERBY("hive.optimize.sampling.orderby.number", 1000, "Total number of samples to be obtained."), diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index b42f966369..80dbbee5ba 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -84,7 +84,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -107,13 +108,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _bucket_number (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col1 (type: string) + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -159,16 +159,15 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _bucket_number - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### - Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -195,6 +194,34 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: id, key + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(id, 'hll') + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Move Operator @@ -236,6 +263,83 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: struct) + auto parallelism: false + Execution mode: vectorized + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index 13d64fb9fa..a8cdc8fe05 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -102,7 +102,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -125,13 +126,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _bucket_number (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col1 (type: string) + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -177,16 +177,15 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _bucket_number - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### - Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -213,6 +212,34 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: id, key + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(id, 'hll') + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Move Operator @@ -254,6 +281,83 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: struct) + auto parallelism: false + Execution mode: vectorized + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out index 7b2561358a..826fae9d42 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -200 +0 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -50 +0 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -100 +0 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -300 +0 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -350 +0 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -400 +50 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -350 +50 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out index 1201ce2107..bb63070503 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -200 +0 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -300 +100 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -400 +200 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -500 +300 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out index 0931e3d557..7758dc9d53 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -200 +0 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -50 +0 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -100 +0 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -300 +0 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -350 +0 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -400 +50 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -350 +50 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out index 15ae3d91a7..0ccd0e4345 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -200 +0 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -300 +100 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -400 +200 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -500 +300 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out index 24fc525804..340791aa7f 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -200 +0 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -50 +0 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -100 +0 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -300 +0 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -350 +0 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -400 +50 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -350 +50 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out index 6bcfe41262..160842236b 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -200 +0 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -300 +100 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -400 +200 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java new file mode 100644 index 0000000000..4c91c49477 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.LlapUtil; +import org.apache.hadoop.hive.ql.optimizer.physical.LlapClusterStateForCompile; +import org.apache.tez.mapreduce.hadoop.MRJobConfig; + +/** + * Contains information about executor memory, various memory thresholds used for join conversions etc. based on + * execution engine. + **/ + +public class MemoryInfo { + + private Configuration conf; + private boolean isTez; + private boolean isLlap; + private long maxExecutorMemory; + private long mapJoinMemoryThreshold; + private long dynPartJoinMemoryThreshold; + + public MemoryInfo(Configuration conf) { + this.isTez = "tez".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)); + this.isLlap = "llap".equalsIgnoreCase(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE)); + if (isLlap) { + LlapClusterStateForCompile llapInfo = LlapClusterStateForCompile.getClusterInfo(conf); + llapInfo.initClusterInfo(); + if (llapInfo.hasClusterInfo()) { + this.maxExecutorMemory = llapInfo.getMemoryPerExecutor(); + } else { + long memPerInstance = + HiveConf.getLongVar(conf, HiveConf.ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB) * 1024L * 1024L; + long numExecutors = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS); + this.maxExecutorMemory = memPerInstance / numExecutors; + } + } else { + if (isTez) { + float heapFraction = HiveConf.getFloatVar(conf, HiveConf.ConfVars.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION); + int containerSizeMb = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? + HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : + conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); + // this can happen when config is explicitly set to "-1", in which case defaultValue also does not work + if (containerSizeMb < 0) { + containerSizeMb = MRJobConfig.DEFAULT_MAP_MEMORY_MB; + } + this.maxExecutorMemory = (long) ((containerSizeMb * 1024L * 1024L) * heapFraction); + } else { + this.maxExecutorMemory = + conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB) * 1024L * 1024L; + // this can happen when config is explicitly set to "-1", in which case defaultValue also does not work + if (maxExecutorMemory < 0) { + maxExecutorMemory = MRJobConfig.DEFAULT_MAP_MEMORY_MB * 1024L * 1024L; + } + } + } + } + + public Configuration getConf() { + return conf; + } + + public void setConf(final Configuration conf) { + this.conf = conf; + } + + public boolean isTez() { + return isTez; + } + + public boolean isLlap() { + return isLlap; + } + + public long getMaxExecutorMemory() { + return maxExecutorMemory; + } + + public long getMapJoinMemoryThreshold() { + return mapJoinMemoryThreshold; + } + + public long getDynPartJoinMemoryThreshold() { + return dynPartJoinMemoryThreshold; + } + + @Override + public String toString() { + return "MEMORY INFO - { isTez: " + isTez() + + ", isLlap: " + isLlap() + + ", maxExecutorMemory: " + LlapUtil.humanReadableByteCount(getMaxExecutorMemory()) + + ", mapJoinMemoryThreshold: "+ LlapUtil.humanReadableByteCount(getMapJoinMemoryThreshold()) + + ", dynPartJoinMemoryThreshold: " + LlapUtil.humanReadableByteCount(getDynPartJoinMemoryThreshold()) + + " }"; + } +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 71f73802c2..25e9cd0482 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -115,12 +115,7 @@ public void initialize(HiveConf hiveConf) { transformations.add(new ConstantPropagate()); } - if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) && - HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") && - HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) && - !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { - transformations.add(new SortedDynPartitionOptimizer()); - } + transformations.add(new SortedDynPartitionTimeGranularityOptimizer()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 314b8b47c2..bf7dbf32b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -27,18 +27,10 @@ import java.util.Set; import java.util.Stack; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.FileSinkOperator; -import org.apache.hadoop.hive.ql.exec.FunctionRegistry; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.OperatorFactory; -import org.apache.hadoop.hive.ql.exec.OperatorUtils; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.RowSchema; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.RecordIdentifier; @@ -56,6 +48,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ColStatistics; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -70,6 +63,7 @@ import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.orc.OrcConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,6 +80,7 @@ public class SortedDynPartitionOptimizer extends Transform { private static final String BUCKET_NUMBER_COL_NAME = "_bucket_number"; + @Override public ParseContext transform(ParseContext pCtx) throws SemanticException { @@ -159,6 +154,12 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } + DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx(); + List partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema()); + + if (!shouldDo(partitionPositions, fsParent)) { + return null; + } // if RS is inserted by enforce bucketing or sorting, we need to remove it // since ReduceSinkDeDuplication will not merge them to single RS. // RS inserted by enforce bucketing/sorting will have bucketing column in @@ -175,13 +176,13 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // unlink connection between FS and its parent fsParent = fsOp.getParentOperators().get(0); + fsParent.getChildOperators().clear(); - DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx(); - int numBuckets = destTable.getNumBuckets(); // if enforce bucketing/sorting is disabled numBuckets will not be set. // set the number of buckets here to ensure creation of empty buckets + int numBuckets = destTable.getNumBuckets(); dpCtx.setNumBuckets(numBuckets); // Get the positions for partition, bucket and sort columns @@ -206,7 +207,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, * which extracts bucketId from it * see {@link org.apache.hadoop.hive.ql.udf.UDFToInteger#evaluate(RecordIdentifier)}*/ ColumnInfo ci = fsParent.getSchema().getSignature().get(0); - if(!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) { + if (!VirtualColumn.ROWID.getTypeInfo().equals(ci.getType())) { throw new IllegalStateException("expected 1st column to be ROW__ID but got wrong type: " + ci.toString()); } //add a cast(ROW__ID as int) to wrap in UDFToInteger() @@ -230,10 +231,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, sortNullOrder.add(order == 1 ? 0 : 1); // for asc, nulls first; for desc, nulls last } LOG.debug("Got sort order"); - for (int i : sortPositions) LOG.debug("sort position " + i); - for (int i : sortOrder) LOG.debug("sort order " + i); - for (int i : sortNullOrder) LOG.debug("sort null order " + i); - List partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema()); + for (int i : sortPositions) { + LOG.debug("sort position " + i); + } + for (int i : sortOrder) { + LOG.debug("sort order " + i); + } + for (int i : sortNullOrder) { + LOG.debug("sort null order " + i); + } // update file sink descriptor fsOp.getConf().setMultiFileSpray(false); @@ -248,7 +254,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // Create ReduceSink operator ReduceSinkOperator rsOp = getReduceSinkOp(partitionPositions, sortPositions, sortOrder, sortNullOrder, - allRSCols, bucketColumns, numBuckets, fsParent, fsOp.getConf().getWriteType()); + allRSCols, bucketColumns, numBuckets, + fsParent, fsOp.getConf().getWriteType()); List descs = new ArrayList(allRSCols.size()); List colNames = new ArrayList(); @@ -265,9 +272,11 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } RowSchema selRS = new RowSchema(fsParent.getSchema()); if (!bucketColumns.isEmpty()) { - descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, ReduceField.KEY.toString()+"."+BUCKET_NUMBER_COL_NAME, null, false)); + descs.add(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, + ReduceField.KEY.toString()+"."+BUCKET_NUMBER_COL_NAME, null, false)); colNames.add(BUCKET_NUMBER_COL_NAME); - ColumnInfo ci = new ColumnInfo(BUCKET_NUMBER_COL_NAME, TypeInfoFactory.stringTypeInfo, selRS.getSignature().get(0).getTabAlias(), true, true); + ColumnInfo ci = new ColumnInfo(BUCKET_NUMBER_COL_NAME, TypeInfoFactory.stringTypeInfo, + selRS.getSignature().get(0).getTabAlias(), true, true); selRS.getSignature().add(ci); fsParent.getSchema().getSignature().add(ci); } @@ -290,7 +299,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } // update partition column info in FS descriptor - fsOp.getConf().setPartitionCols( rsOp.getConf().getPartitionCols()); + fsOp.getConf().setPartitionCols(rsOp.getConf().getPartitionCols()); LOG.info("Inserted " + rsOp.getOperatorId() + " and " + selOp.getOperatorId() + " as parent of " + fsOp.getOperatorId() + " and child of " + fsParent.getOperatorId()); @@ -478,7 +487,7 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, String orderStr = ""; for (Integer i : newSortOrder) { - if(i == 1) { + if (i == 1) { orderStr += "+"; } else { orderStr += "-"; @@ -502,7 +511,7 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, String nullOrderStr = ""; for (Integer i : newSortNullOrder) { - if(i == 0) { + if (i == 0) { nullOrderStr += "a"; } else { nullOrderStr += "z"; @@ -517,7 +526,7 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, for (Integer idx : keyColsPosInVal) { if (idx < 0) { ExprNodeDesc bucketNumColUDF = ExprNodeGenericFuncDesc.newInstance( - FunctionRegistry.getFunctionInfo("bucket_number").getGenericUDF(), new ArrayList<>()); + FunctionRegistry.getFunctionInfo("bucket_number").getGenericUDF(), new ArrayList<>()); keyCols.add(bucketNumColUDF); colExprMap.put(Utilities.ReduceField.KEY + "." +BUCKET_NUMBER_COL_NAME, bucketNumColUDF); } else { @@ -598,7 +607,8 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, } /** - * Get the sort positions for the sort columns + * Get the sort positions for the sort columns. + * * @param tabSortCols * @param tabCols * @return @@ -620,7 +630,8 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, } /** - * Get the sort order for the sort columns + * Get the sort order for the sort columns. + * * @param tabSortCols * @param tabCols * @return @@ -652,6 +663,63 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, return cols; } - } + // the idea is to estimate how many number of writers this insert can spun up. + // Writers are proportional to number of partitions being inserted i.e cardinality of the partition columns + // if these writers are less than number of writers allowed within the memory pool (estimated) we go ahead with + // adding extra RS + // The way max number of writers allowed are computed based on + // (executor/container memory) * (percentage of memory taken by orc) + // and dividing that by max memory (stripe size) taken by a single writer. + private boolean shouldDo(List partitionPos, Operator fsParent) { + + int threshold = HiveConf.getIntVar(this.parseCtx.getConf(), + HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD); + long MAX_WRITERS = -1; + + switch (threshold) { + case -1: + return false; + case 0: + break; + case 1: + return true; + default: + MAX_WRITERS = threshold; + break; + } + + List colStats = fsParent.getStatistics().getColumnStats(); + if (colStats == null || colStats.isEmpty()) { + return true; + } + long partCardinality = 1; + + // compute cardinality for partition columns + for (Integer idx : partitionPos) { + ColumnInfo ci = fsParent.getSchema().getSignature().get(idx); + ColStatistics partStats = fsParent.getStatistics().getColumnStatisticsFromColName(ci.getInternalName()); + if (partStats == null) { + // statistics for this partition are for some reason not available + return true; + } + partCardinality = partCardinality * partStats.getCountDistint(); + } + + if (MAX_WRITERS < 0) { + double orcMemPool = this.parseCtx.getConf().getDouble(OrcConf.MEMORY_POOL.getHiveConfName(), + (Double) OrcConf.MEMORY_POOL.getDefaultValue()); + long orcStripSize = this.parseCtx.getConf().getLong(OrcConf.STRIPE_SIZE.getHiveConfName(), + (Long) OrcConf.STRIPE_SIZE.getDefaultValue()); + MemoryInfo memoryInfo = new MemoryInfo(this.parseCtx.getConf()); + LOG.debug("Memory info during SDPO opt: {}", memoryInfo); + long executorMem = memoryInfo.getMaxExecutorMemory(); + MAX_WRITERS = (long) (executorMem * orcMemPool) / orcStripSize; + } + if (partCardinality <= MAX_WRITERS) { + return false; + } + return true; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java index 0d1990a8cb..a15bdd5e96 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java @@ -44,6 +44,7 @@ private volatile Long lastClusterUpdateNs; private volatile Integer noConfigNodeCount, executorCount; private volatile int numExecutorsPerNode = -1; + private volatile long memoryPerInstance = -1; private LlapRegistryService svc; private final Configuration conf; private final long updateIntervalNs; @@ -93,6 +94,14 @@ public int getNumExecutorsPerNode() { return numExecutorsPerNode; } + public long getMemoryPerInstance() { + return memoryPerInstance; + } + + public long getMemoryPerExecutor() { + return getMemoryPerInstance() / getNumExecutorsPerNode(); + } + private boolean isUpdateNeeded() { Long lastUpdateLocal = lastClusterUpdateNs; if (lastUpdateLocal == null) return true; @@ -134,6 +143,9 @@ public boolean initClusterInfo() { if (numExecutorsPerNode == -1) { numExecutorsPerNode = numExecutors; } + if (memoryPerInstance == -1) { + memoryPerInstance = si.getResource().getMemorySize() * 1024L * 1024L; + } } catch (NumberFormatException e) { ++noConfigNodesLocal; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index fdc963506c..91d2f1f091 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -77,11 +77,14 @@ import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin; import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization; import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc; +import org.apache.hadoop.hive.ql.optimizer.NonBlockingOpDeDupProc; import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc; import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize; import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism; import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer; +import org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer; import org.apache.hadoop.hive.ql.optimizer.TopNKeyProcessor; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication; import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkJoinDeDuplication; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; @@ -170,6 +173,27 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // Update bucketing version of ReduceSinkOp if needed updateBucketingVersionForUpgrade(procCtx); + // run Sorted dynamic partition optimization + if(HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.DYNAMICPARTITIONING) && + HiveConf.getVar(procCtx.conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") && + !HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + new SortedDynPartitionOptimizer().transform(procCtx.parseContext); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Sorted dynamic partition optimization"); + } + + if(HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION) + || procCtx.parseContext.hasAcidWrite()) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + // Dynamic sort partition adds an extra RS therefore need to de-dup + new ReduceSinkDeDuplication().transform(procCtx.parseContext); + // there is an issue with dedup logic wherein SELECT is created with wrong columns + // NonBlockingOpDeDupProc fixes that + // (kind of hackish, the issue in de-dup should be fixed but it needs more investigation) + new NonBlockingOpDeDupProc().transform(procCtx.parseContext); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Reduce Sink de-duplication"); + } + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // run the optimizations that use stats for optimization runStatsDependentOptimizations(procCtx, inputs, outputs); diff --git a/ql/src/test/queries/clientpositive/dp_counter_mm.q b/ql/src/test/queries/clientpositive/dp_counter_mm.q index 91c4f4293c..1c655fd79c 100644 --- a/ql/src/test/queries/clientpositive/dp_counter_mm.q +++ b/ql/src/test/queries/clientpositive/dp_counter_mm.q @@ -4,6 +4,7 @@ set hive.exec.max.dynamic.partitions.pernode=200; set hive.exec.max.dynamic.partitions=200; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.optimize.sort.dynamic.partition.threshold=-1; drop table src2_n5; create table src2_n5 (key int) partitioned by (value string) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only"); diff --git a/ql/src/test/queries/clientpositive/dp_counter_non_mm.q b/ql/src/test/queries/clientpositive/dp_counter_non_mm.q index 561ae6e8b3..d2da0b0e15 100644 --- a/ql/src/test/queries/clientpositive/dp_counter_non_mm.q +++ b/ql/src/test/queries/clientpositive/dp_counter_non_mm.q @@ -2,6 +2,7 @@ set hive.exec.dynamic.partition.mode=nonstrict; set hive.exec.max.dynamic.partitions.pernode=200; set hive.exec.max.dynamic.partitions=200; +set hive.optimize.sort.dynamic.partition.threshold=-1; drop table src2_n3; create table src2_n3 (key int) partitioned by (value string); diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q index 3c2918fdca..8a3522de0b 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q @@ -8,8 +8,7 @@ set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; set hive.exec.dynamic.partition.mode=nonstrict; set hive.vectorized.execution.enabled=true; - - +set hive.optimize.sort.dynamic.partition.threshold=1; create table over1k_n1( t tinyint, @@ -106,19 +105,19 @@ create table over1k_part2_orc( f float) partitioned by (ds string, t tinyint); -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i; explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_orc order by i limit 10) tmp where t is null or t=27; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 group by si,i,b,f,t; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; -- tests for HIVE-8162, only partition column 't' should be in last RS operator explain insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 group by si,i,b,f,t; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i; desc formatted over1k_part2_orc partition(ds="foo",t=27); @@ -128,7 +127,7 @@ desc formatted over1k_part2_orc partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__ select * from over1k_part2_orc; select count(*) from over1k_part2_orc; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t from over1k_orc where t is null or t=27 order by i; desc formatted over1k_part2_orc partition(ds="foo",t=27); @@ -150,12 +149,12 @@ create table over1k_part_buck_sort2_orc( clustered by (si) sorted by (f) into 1 buckets; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27; desc formatted over1k_part_buck_sort2_orc partition(t=27); @@ -166,7 +165,7 @@ select * from over1k_part_buck_sort2_orc; explain select count(*) from over1k_part_buck_sort2_orc; select count(*) from over1k_part_buck_sort2_orc; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; insert overwrite table over1k_part_buck_sort2_orc partition(t) select si,i,b,f,t from over1k_orc where t is null or t=27; desc formatted over1k_part_buck_sort2_orc partition(t=27); @@ -194,7 +193,7 @@ partitioned by (s string) clustered by (si) into 2 buckets stored as orc tblproperties ('transactional'='true'); -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert into table addcolumns_vectorization_true_disallowincompatible_true_fileformat_orc_tinyint partition (s) select cint,csmallint, cstring1 from alltypesorc limit 10; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q index 436c0edc77..dbeb874019 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q @@ -7,8 +7,7 @@ set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; set hive.exec.dynamic.partition.mode=nonstrict; - - +set hive.optimize.sort.dynamic.partition.threshold=1; create table over1k_n3( t tinyint, @@ -100,19 +99,19 @@ create table over1k_part2( f float) partitioned by (ds string, t tinyint); -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i; explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from (select * from over1k_n3 order by i limit 10) tmp where t is null or t=27; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; -- tests for HIVE-8162, only partition column 't' should be in last RS operator explain insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 group by si,i,b,f,t; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i; desc formatted over1k_part2 partition(ds="foo",t=27); @@ -122,7 +121,7 @@ desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__"); select * from over1k_part2; select count(*) from over1k_part2; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from over1k_n3 where t is null or t=27 order by i; desc formatted over1k_part2 partition(ds="foo",t=27); @@ -144,12 +143,12 @@ create table over1k_part_buck_sort2( clustered by (si) sorted by (f) into 1 buckets; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27; desc formatted over1k_part_buck_sort2 partition(t=27); @@ -158,7 +157,7 @@ desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__"); select * from over1k_part_buck_sort2; select count(*) from over1k_part_buck_sort2; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; insert overwrite table over1k_part_buck_sort2 partition(t) select si,i,b,f,t from over1k_n3 where t is null or t=27; desc formatted over1k_part_buck_sort2 partition(t=27); @@ -173,7 +172,7 @@ create table over1k_part3( f float) partitioned by (s string, t tinyint, i int); -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where s="foo"; explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where t=27; explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100; @@ -209,3 +208,41 @@ insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from ov insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo"; select sum(hash(*)) from over1k_part3; + +drop table over1k_n3; +create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|'; + +load data local inpath '../../data/files/over1k' into table over1k_n3; + +analyze table over1k_n3 compute statistics for columns; +set hive.stats.fetch.column.stats=true; + +-- default hive should do cost based and add extra RS +set hive.optimize.sort.dynamic.partition.threshold=0; +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27; + +-- default but shouldn't add extra RS +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10; + +-- disable +set hive.optimize.sort.dynamic.partition.threshold=-1; +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27; + +-- enable, will add extra RS +set hive.optimize.sort.dynamic.partition.threshold=1; +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10; + +drop table over1k_n3; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q index 6dfb51a7da..2f3afb6d0e 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q @@ -6,8 +6,7 @@ set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; set hive.exec.dynamic.partition.mode=nonstrict; - - +set hive.optimize.sort.dynamic.partition.threshold=1; -- SORT_QUERY_RESULTS @@ -78,7 +77,7 @@ select * from ss_part where ss_sold_date_sk=2452617; desc formatted ss_part partition(ss_sold_date_sk=2452638); select * from ss_part where ss_sold_date_sk=2452638; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; -- SORT DYNAMIC PARTITION DISABLED explain insert overwrite table ss_part partition (ss_sold_date_sk) @@ -210,7 +209,7 @@ create table if not exists hive13_dp1 ( PARTITIONED BY(`day` string) STORED AS ORC; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; explain insert overwrite table `hive13_dp1` partition(`day`) select key k1, @@ -228,7 +227,7 @@ from src group by "day", key; select * from hive13_dp1 order by k1, k2 limit 5; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; explain insert overwrite table `hive13_dp1` partition(`day`) select key k1, diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q index d0b32cde0d..7e27b88304 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid.q @@ -7,7 +7,7 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.exec.dynamic.partition.mode=nonstrict; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; -- single level partition, sorted dynamic partition disabled drop table if exists acid_part; @@ -31,7 +31,7 @@ select count(*) from acid_part where ds in ('2008-04-08'); delete from acid_part where key = 'foo' and ds='2008-04-08'; select count(*) from acid_part where ds='2008-04-08'; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; -- single level partition, sorted dynamic partition enabled drop table if exists acid_part_sdpo; @@ -54,7 +54,7 @@ select count(*) from acid_part_sdpo where ds in ('2008-04-08'); delete from acid_part_sdpo where key = 'foo' and ds='2008-04-08'; select count(*) from acid_part_sdpo where ds='2008-04-08'; -set hive.optimize.sort.dynamic.partition=false; +set hive.optimize.sort.dynamic.partition.threshold=-1; -- 2 level partition, sorted dynamic partition disabled drop table if exists acid_2L_part; @@ -83,7 +83,7 @@ delete from acid_2L_part where value = 'bar'; delete from acid_2L_part where value = 'bar'; select count(*) from acid_2L_part; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; -- 2 level partition, sorted dynamic partition enabled drop table if exists acid_2L_part_sdpo; @@ -113,7 +113,7 @@ delete from acid_2L_part_sdpo where value = 'bar'; select count(*) from acid_2L_part_sdpo; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; set hive.optimize.constant.propagation=false; -- 2 level partition, sorted dynamic partition enabled, constant propagation disabled @@ -137,4 +137,4 @@ select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr>=11; delete from acid_2L_part_sdpo_no_cp where key = 'foo' and ds='2008-04-08' and hr=11; select count(*) from acid_2L_part_sdpo_no_cp where ds='2008-04-08' and hr=11; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q index c852d519d8..ca8a44284c 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization_acid2.q @@ -2,7 +2,7 @@ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.exec.dynamic.partition.mode=nonstrict; -set hive.optimize.sort.dynamic.partition=true; +set hive.optimize.sort.dynamic.partition.threshold=1; CREATE TABLE non_acid(key string, value string) PARTITIONED BY(ds string, hr int) diff --git a/ql/src/test/queries/clientpositive/load_data_using_job.q b/ql/src/test/queries/clientpositive/load_data_using_job.q index 970a7521f5..a2d45e52bc 100644 --- a/ql/src/test/queries/clientpositive/load_data_using_job.q +++ b/ql/src/test/queries/clientpositive/load_data_using_job.q @@ -9,6 +9,8 @@ set hive.auto.convert.join.noconditionaltask=true; set hive.auto.convert.join.noconditionaltask.size=10000; set hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.TableSizeBasedBigTableSelectorForAutoSMJ; +-- SORT_QUERY_RESULTS + -- Single partition -- Regular load happens. CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string) STORED AS TEXTFILE; diff --git a/ql/src/test/queries/clientpositive/tez_input_counters.q b/ql/src/test/queries/clientpositive/tez_input_counters.q index 44619662b9..1d9d7de9b3 100644 --- a/ql/src/test/queries/clientpositive/tez_input_counters.q +++ b/ql/src/test/queries/clientpositive/tez_input_counters.q @@ -8,6 +8,7 @@ set hive.fetch.task.conversion=none; set hive.map.aggr=false; -- disabling map side aggregation as that can lead to different intermediate record counts set hive.tez.exec.print.summary=true; +set hive.optimize.sort.dynamic.partition.threshold=-1; create table testpart (k int) partitioned by (v string); insert overwrite table testpart partition(v) select * from src; diff --git a/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q b/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q index 74ca011401..eaa714ee69 100644 --- a/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q +++ b/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q @@ -4,7 +4,6 @@ set hive.fetch.task.conversion=none; -- Check if vectorization code is handling partitioning on DATE and the other data types. - CREATE TABLE flights_tiny_n1 ( origin_city_name STRING, dest_city_name STRING, @@ -19,6 +18,8 @@ CREATE TABLE flights_tiny_orc STORED AS ORC AS SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num FROM flights_tiny_n1; +-- SORT_QUERY_RESULTS + SELECT * FROM flights_tiny_orc; SET hive.vectorized.execution.enabled=false; diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out index e1af88ad81..e06624c32b 100644 --- a/ql/src/test/results/clientpositive/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -97,7 +97,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4063 + totalSize 4542 #### A masked pattern was here #### # Storage Information @@ -184,7 +184,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4063 + totalSize 4542 #### A masked pattern was here #### # Storage Information @@ -235,7 +235,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4063 + totalSize 4542 #### A masked pattern was here #### # Storage Information @@ -331,7 +331,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8126 + totalSize 9085 #### A masked pattern was here #### # Storage Information @@ -380,7 +380,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8126 + totalSize 9085 #### A masked pattern was here #### # Storage Information @@ -455,11 +455,11 @@ STAGE PLANS: TableScan alias: acid filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(key) mode: hash @@ -591,7 +591,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 176000 - totalSize 3008 + totalSize 3485 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 20f88bdb70..5bc944271d 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -31,8 +31,10 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -46,26 +48,62 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: string), _col3 (type: string), _bucket_number (type: string), _col1 (type: string) - sort order: ++++ - Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + key expressions: _col1 (type: string) + sort order: + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), UDFToInteger(_col3) (type: int) + outputColumnNames: key, value, ds, hr + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string), hr (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Move Operator @@ -80,7 +118,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -88,3 +126,33 @@ STAGE PLANS: Column Types: string, string Table: default.non_acid + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out index 0414812ca9..2c6d7cafa7 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out @@ -223,16 +223,16 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: pri BETWEEN 1 AND 2 (type: boolean) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs @@ -246,14 +246,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -371,11 +371,11 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: pri BETWEEN 1 AND 2 (type: boolean) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -392,14 +392,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -627,11 +627,11 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: pri BETWEEN 1 AND 2 (type: boolean) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -648,14 +648,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index 62fdbdba42..bd3c7769a4 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -189,7 +189,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col0 (type: smallint) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -199,7 +198,6 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -275,7 +273,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -293,7 +290,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap @@ -301,7 +297,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -374,7 +369,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -384,11 +378,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -456,7 +449,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -466,11 +458,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -606,7 +597,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col0 (type: smallint) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -616,7 +606,6 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -692,7 +681,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -710,7 +698,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap @@ -718,7 +705,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -791,7 +777,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -801,11 +786,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -873,7 +857,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -883,11 +866,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1171,7 +1153,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4648 + totalSize 4670 #### A masked pattern was here #### # Storage Information @@ -1249,7 +1231,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4658 + totalSize 4656 #### A masked pattern was here #### # Storage Information @@ -1527,7 +1509,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -1537,7 +1518,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1607,7 +1587,6 @@ STAGE PLANS: key expressions: _col2 (type: int) sort order: + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -1632,7 +1611,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap @@ -1640,7 +1618,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2347,7 +2324,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -2357,11 +2333,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2906,7 +2881,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs @@ -2924,7 +2898,6 @@ STAGE PLANS: key expressions: _col2 (type: string), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: smallint) Reducer 3 Execution mode: vectorized, llap @@ -2932,11 +2905,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: smallint), KEY._col2 (type: string), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _bucket_number - Statistics: Num rows: 10 Data size: 2656 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 10 Data size: 2656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index ec95389d08..bf29a112fc 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -146,7 +146,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -156,7 +155,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -232,7 +230,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs @@ -250,7 +247,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: llap @@ -258,7 +254,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -331,7 +326,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -341,7 +335,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -413,7 +406,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -423,7 +415,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -563,7 +554,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -573,7 +563,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -649,7 +638,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs @@ -667,7 +655,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: llap @@ -675,7 +662,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -748,7 +734,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -758,7 +743,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -830,7 +814,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -840,7 +823,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -1484,7 +1466,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -1494,7 +1475,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1564,7 +1544,6 @@ STAGE PLANS: key expressions: _col2 (type: int) sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: llap LLAP IO: no inputs @@ -1589,7 +1568,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: llap @@ -1597,7 +1575,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2304,7 +2281,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -2314,7 +2290,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -2680,7 +2655,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2690,7 +2664,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2764,7 +2737,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2774,7 +2746,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2848,7 +2819,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2858,7 +2828,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2932,7 +2901,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2942,7 +2910,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -3016,7 +2983,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -3026,7 +2992,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -3100,7 +3065,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -3110,7 +3074,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -3584,3 +3547,455 @@ POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 #### A masked pattern was here #### 17814641134 +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: analyze table over1k_n3 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +POSTHOOK: query: analyze table over1k_n3 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 352 Data size: 39072 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index d953eebefd..30074abaf2 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -352,19 +352,17 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) - sort order: + + sort order: ++ Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) + value expressions: _col0 (type: float), _col1 (type: float) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), VALUE._col2 (type: int) + expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), KEY._col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 5fd1bf6fb2..40dc5e97e5 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -515,20 +515,19 @@ STAGE PLANS: key expressions: _col3 (type: string), _bucket_number (type: string), _col0 (type: struct) sort order: +++ Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: 'foo' (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) + expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number - Statistics: Num rows: 5 Data size: 2220 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2220 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1286,20 +1285,19 @@ STAGE PLANS: key expressions: '2008-04-08' (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) sort order: ++++ Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int) - Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: 'foo' (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) + expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1431,7 +1429,6 @@ STAGE PLANS: key expressions: _col1 (type: string), _col2 (type: int), _bucket_number (type: string), _col0 (type: struct) sort order: ++++ Map-reduce partition columns: _col1 (type: string), _col2 (type: int) - Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 @@ -1440,11 +1437,10 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: struct), KEY._col1 (type: string), KEY._col2 (type: int), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _bucket_number - Statistics: Num rows: 5 Data size: 2240 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2240 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1636,7 +1632,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) sort order: ++++ Map-reduce partition columns: _col3 (type: string), _col4 (type: int) - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1646,11 +1641,10 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1745,7 +1739,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) sort order: ++++ Map-reduce partition columns: _col3 (type: string), _col4 (type: int) - Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col1 (type: string), 'bar' (type: string) Execution mode: llap LLAP IO: may be used (ACID table) @@ -1755,11 +1748,10 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out index 9c7babba0e..916db931b6 100644 --- a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out +++ b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out @@ -299,7 +299,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 545 + totalSize 539 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -391,7 +391,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 595 + totalSize 594 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -690,7 +690,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 545 + totalSize 539 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde @@ -782,7 +782,7 @@ STAGE PLANS: serialization.ddl struct loc_orc_1d_n0 { string state, double locid, decimal(10,0) cnt, i32 zip} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 595 + totalSize 594 #### A masked pattern was here #### serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out b/ql/src/test/results/clientpositive/llap/lineage3.q.out index cf38816127..9bec309c9c 100644 --- a/ql/src/test/results/clientpositive/llap/lineage3.q.out +++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out @@ -372,7 +372,7 @@ PREHOOK: query: create table src_dp1 (f string, w string, m int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@src_dp1 -Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[62][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: from src_dp, src_dp1 insert into dest_dp1 partition (year) select first, word, year insert into dest_dp2 partition (y, m) select first, word, year, month diff --git a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index 7e6e88bd08..18dca4921f 100644 --- a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -1916,7 +1916,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: oft - Statistics: Num rows: 12288 Data size: 1884148 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 13243096 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true vectorizationSchemaColumns: [0:csmallint:smallint, 1:cint:int, 2:cbigint:bigint, 3:cfloat:float, 4:cdouble:double, 5:cstring1:string, 6:cchar1:char(255), 7:cvchar1:varchar(255), 8:cboolean1:boolean, 9:cboolean2:boolean, 10:ctinyint:tinyint, 11:ROW__ID:struct] @@ -1939,7 +1939,7 @@ STAGE PLANS: outputColumnNames: _col1, _col6, _col7, _col10 input vertices: 1 Map 2 - Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col10 (type: tinyint), _col1 (type: int), _col6 (type: char(255)), _col7 (type: varchar(255)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -1947,13 +1947,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [10, 1, 6, 7] - Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/llap_smb.q.out b/ql/src/test/results/clientpositive/llap/llap_smb.q.out index ed10999f8f..d2593b19c9 100644 --- a/ql/src/test/results/clientpositive/llap/llap_smb.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_smb.q.out @@ -269,10 +269,10 @@ STAGE PLANS: TableScan alias: a filterExpr: id is not null (type: boolean) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: id is not null (type: boolean) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Merge Join Operator condition map: Inner Join 0 to 1 @@ -280,18 +280,18 @@ STAGE PLANS: 0 id (type: bigint) 1 id (type: bigint) outputColumnNames: _col2, _col3 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() keys: _col2 (type: int), _col3 (type: smallint) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: smallint) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: smallint) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: bigint) Execution mode: llap Reducer 2 @@ -302,10 +302,10 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: smallint) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out index 406c6204e5..7ad7bfbed4 100644 --- a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out +++ b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out @@ -54,124 +54,124 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 #### A masked pattern was here #### -165 val_165 2008-04-08 -484 val_484 2008-04-08 -150 val_150 2008-04-08 -224 val_224 2008-04-08 -66 val_66 2008-04-08 -213 val_213 2008-04-08 -374 val_374 2008-04-08 -495 val_495 2008-04-08 -37 val_37 2008-04-08 -327 val_327 2008-04-08 -15 val_15 2008-04-08 -338 val_338 2008-04-08 -459 val_459 2008-04-08 -466 val_466 2008-04-08 -396 val_396 2008-04-08 -309 val_309 2008-04-08 -367 val_367 2008-04-08 0 val_0 2008-04-08 -455 val_455 2008-04-08 -316 val_316 2008-04-08 -345 val_345 2008-04-08 +0 val_0 2008-04-08 +0 val_0 2008-04-08 +103 val_103 2008-04-08 +103 val_103 2008-04-08 +11 val_11 2008-04-08 +114 val_114 2008-04-08 +118 val_118 2008-04-08 +118 val_118 2008-04-08 +125 val_125 2008-04-08 +125 val_125 2008-04-08 129 val_129 2008-04-08 -378 val_378 2008-04-08 -4 val_4 2008-04-08 -356 val_356 2008-04-08 +129 val_129 2008-04-08 +136 val_136 2008-04-08 +143 val_143 2008-04-08 +15 val_15 2008-04-08 +15 val_15 2008-04-08 +150 val_150 2008-04-08 +158 val_158 2008-04-08 +165 val_165 2008-04-08 +165 val_165 2008-04-08 169 val_169 2008-04-08 -125 val_125 2008-04-08 -437 val_437 2008-04-08 -286 val_286 2008-04-08 -187 val_187 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +172 val_172 2008-04-08 +172 val_172 2008-04-08 176 val_176 2008-04-08 -459 val_459 2008-04-08 -51 val_51 2008-04-08 -103 val_103 2008-04-08 -239 val_239 2008-04-08 -213 val_213 2008-04-08 176 val_176 2008-04-08 -275 val_275 2008-04-08 -260 val_260 2008-04-08 -404 val_404 2008-04-08 +183 val_183 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +19 val_19 2008-04-08 +190 val_190 2008-04-08 +194 val_194 2008-04-08 +202 val_202 2008-04-08 +213 val_213 2008-04-08 +213 val_213 2008-04-08 217 val_217 2008-04-08 -84 val_84 2008-04-08 -466 val_466 2008-04-08 -8 val_8 2008-04-08 -411 val_411 2008-04-08 -172 val_172 2008-04-08 -129 val_129 2008-04-08 -158 val_158 2008-04-08 -0 val_0 2008-04-08 +217 val_217 2008-04-08 +224 val_224 2008-04-08 +224 val_224 2008-04-08 +228 val_228 2008-04-08 +235 val_235 2008-04-08 +239 val_239 2008-04-08 +239 val_239 2008-04-08 +242 val_242 2008-04-08 +242 val_242 2008-04-08 +257 val_257 2008-04-08 26 val_26 2008-04-08 -165 val_165 2008-04-08 -327 val_327 2008-04-08 -51 val_51 2008-04-08 -404 val_404 2008-04-08 -95 val_95 2008-04-08 +26 val_26 2008-04-08 +260 val_260 2008-04-08 +275 val_275 2008-04-08 282 val_282 2008-04-08 -187 val_187 2008-04-08 -316 val_316 2008-04-08 -169 val_169 2008-04-08 -77 val_77 2008-04-08 -0 val_0 2008-04-08 -118 val_118 2008-04-08 282 val_282 2008-04-08 -419 val_419 2008-04-08 -15 val_15 2008-04-08 -118 val_118 2008-04-08 -19 val_19 2008-04-08 -224 val_224 2008-04-08 +286 val_286 2008-04-08 +305 val_305 2008-04-08 309 val_309 2008-04-08 -389 val_389 2008-04-08 +309 val_309 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +323 val_323 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 327 val_327 2008-04-08 -242 val_242 2008-04-08 -392 val_392 2008-04-08 -242 val_242 2008-04-08 -396 val_396 2008-04-08 -95 val_95 2008-04-08 -11 val_11 2008-04-08 -143 val_143 2008-04-08 -228 val_228 2008-04-08 33 val_33 2008-04-08 -103 val_103 2008-04-08 +338 val_338 2008-04-08 +341 val_341 2008-04-08 +345 val_345 2008-04-08 +356 val_356 2008-04-08 367 val_367 2008-04-08 -239 val_239 2008-04-08 -480 val_480 2008-04-08 -202 val_202 2008-04-08 -316 val_316 2008-04-08 -235 val_235 2008-04-08 -80 val_80 2008-04-08 -44 val_44 2008-04-08 -466 val_466 2008-04-08 -257 val_257 2008-04-08 -190 val_190 2008-04-08 -114 val_114 2008-04-08 +367 val_367 2008-04-08 +37 val_37 2008-04-08 +37 val_37 2008-04-08 +374 val_374 2008-04-08 +378 val_378 2008-04-08 +389 val_389 2008-04-08 +392 val_392 2008-04-08 396 val_396 2008-04-08 -217 val_217 2008-04-08 -125 val_125 2008-04-08 -187 val_187 2008-04-08 -480 val_480 2008-04-08 -491 val_491 2008-04-08 -305 val_305 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +4 val_4 2008-04-08 +400 val_400 2008-04-08 +404 val_404 2008-04-08 +404 val_404 2008-04-08 +411 val_411 2008-04-08 +419 val_419 2008-04-08 +437 val_437 2008-04-08 +44 val_44 2008-04-08 444 val_444 2008-04-08 -169 val_169 2008-04-08 -323 val_323 2008-04-08 -480 val_480 2008-04-08 -136 val_136 2008-04-08 -172 val_172 2008-04-08 +448 val_448 2008-04-08 +455 val_455 2008-04-08 +459 val_459 2008-04-08 +459 val_459 2008-04-08 462 val_462 2008-04-08 -26 val_26 2008-04-08 462 val_462 2008-04-08 -341 val_341 2008-04-08 -183 val_183 2008-04-08 -84 val_84 2008-04-08 -37 val_37 2008-04-08 -448 val_448 2008-04-08 -194 val_194 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 477 val_477 2008-04-08 -169 val_169 2008-04-08 -400 val_400 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +484 val_484 2008-04-08 +491 val_491 2008-04-08 +495 val_495 2008-04-08 +51 val_51 2008-04-08 +51 val_51 2008-04-08 +66 val_66 2008-04-08 +77 val_77 2008-04-08 +8 val_8 2008-04-08 +80 val_80 2008-04-08 +84 val_84 2008-04-08 +84 val_84 2008-04-08 +95 val_95 2008-04-08 +95 val_95 2008-04-08 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -204,6 +204,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -215,16 +218,28 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -265,124 +280,124 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 #### A masked pattern was here #### -165 val_165 2008-04-08 -484 val_484 2008-04-08 -150 val_150 2008-04-08 -224 val_224 2008-04-08 -66 val_66 2008-04-08 -213 val_213 2008-04-08 -374 val_374 2008-04-08 -495 val_495 2008-04-08 -37 val_37 2008-04-08 -327 val_327 2008-04-08 -15 val_15 2008-04-08 -338 val_338 2008-04-08 -459 val_459 2008-04-08 -466 val_466 2008-04-08 -396 val_396 2008-04-08 -309 val_309 2008-04-08 -367 val_367 2008-04-08 0 val_0 2008-04-08 -455 val_455 2008-04-08 -316 val_316 2008-04-08 -345 val_345 2008-04-08 -129 val_129 2008-04-08 -378 val_378 2008-04-08 -4 val_4 2008-04-08 -356 val_356 2008-04-08 -169 val_169 2008-04-08 -125 val_125 2008-04-08 -437 val_437 2008-04-08 -286 val_286 2008-04-08 -187 val_187 2008-04-08 -176 val_176 2008-04-08 -459 val_459 2008-04-08 -51 val_51 2008-04-08 +0 val_0 2008-04-08 +0 val_0 2008-04-08 103 val_103 2008-04-08 -239 val_239 2008-04-08 -213 val_213 2008-04-08 -176 val_176 2008-04-08 -275 val_275 2008-04-08 -260 val_260 2008-04-08 -404 val_404 2008-04-08 -217 val_217 2008-04-08 -84 val_84 2008-04-08 -466 val_466 2008-04-08 -8 val_8 2008-04-08 -411 val_411 2008-04-08 -172 val_172 2008-04-08 +103 val_103 2008-04-08 +11 val_11 2008-04-08 +114 val_114 2008-04-08 +118 val_118 2008-04-08 +118 val_118 2008-04-08 +125 val_125 2008-04-08 +125 val_125 2008-04-08 +129 val_129 2008-04-08 129 val_129 2008-04-08 +136 val_136 2008-04-08 +143 val_143 2008-04-08 +15 val_15 2008-04-08 +15 val_15 2008-04-08 +150 val_150 2008-04-08 158 val_158 2008-04-08 -0 val_0 2008-04-08 -26 val_26 2008-04-08 165 val_165 2008-04-08 -327 val_327 2008-04-08 -51 val_51 2008-04-08 -404 val_404 2008-04-08 -95 val_95 2008-04-08 -282 val_282 2008-04-08 -187 val_187 2008-04-08 -316 val_316 2008-04-08 +165 val_165 2008-04-08 169 val_169 2008-04-08 -77 val_77 2008-04-08 -0 val_0 2008-04-08 -118 val_118 2008-04-08 -282 val_282 2008-04-08 -419 val_419 2008-04-08 -15 val_15 2008-04-08 -118 val_118 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +172 val_172 2008-04-08 +172 val_172 2008-04-08 +176 val_176 2008-04-08 +176 val_176 2008-04-08 +183 val_183 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 19 val_19 2008-04-08 +190 val_190 2008-04-08 +194 val_194 2008-04-08 +202 val_202 2008-04-08 +213 val_213 2008-04-08 +213 val_213 2008-04-08 +217 val_217 2008-04-08 +217 val_217 2008-04-08 +224 val_224 2008-04-08 224 val_224 2008-04-08 -309 val_309 2008-04-08 -389 val_389 2008-04-08 -327 val_327 2008-04-08 -242 val_242 2008-04-08 -392 val_392 2008-04-08 -242 val_242 2008-04-08 -396 val_396 2008-04-08 -95 val_95 2008-04-08 -11 val_11 2008-04-08 -143 val_143 2008-04-08 228 val_228 2008-04-08 -33 val_33 2008-04-08 -103 val_103 2008-04-08 -367 val_367 2008-04-08 -239 val_239 2008-04-08 -480 val_480 2008-04-08 -202 val_202 2008-04-08 -316 val_316 2008-04-08 235 val_235 2008-04-08 -80 val_80 2008-04-08 -44 val_44 2008-04-08 -466 val_466 2008-04-08 +239 val_239 2008-04-08 +239 val_239 2008-04-08 +242 val_242 2008-04-08 +242 val_242 2008-04-08 257 val_257 2008-04-08 -190 val_190 2008-04-08 -114 val_114 2008-04-08 -396 val_396 2008-04-08 -217 val_217 2008-04-08 -125 val_125 2008-04-08 -187 val_187 2008-04-08 -480 val_480 2008-04-08 -491 val_491 2008-04-08 +26 val_26 2008-04-08 +26 val_26 2008-04-08 +260 val_260 2008-04-08 +275 val_275 2008-04-08 +282 val_282 2008-04-08 +282 val_282 2008-04-08 +286 val_286 2008-04-08 305 val_305 2008-04-08 -444 val_444 2008-04-08 -169 val_169 2008-04-08 +309 val_309 2008-04-08 +309 val_309 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 323 val_323 2008-04-08 -480 val_480 2008-04-08 -136 val_136 2008-04-08 -172 val_172 2008-04-08 -462 val_462 2008-04-08 -26 val_26 2008-04-08 -462 val_462 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 +33 val_33 2008-04-08 +338 val_338 2008-04-08 341 val_341 2008-04-08 -183 val_183 2008-04-08 -84 val_84 2008-04-08 +345 val_345 2008-04-08 +356 val_356 2008-04-08 +367 val_367 2008-04-08 +367 val_367 2008-04-08 37 val_37 2008-04-08 +37 val_37 2008-04-08 +374 val_374 2008-04-08 +378 val_378 2008-04-08 +389 val_389 2008-04-08 +392 val_392 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +4 val_4 2008-04-08 +400 val_400 2008-04-08 +404 val_404 2008-04-08 +404 val_404 2008-04-08 +411 val_411 2008-04-08 +419 val_419 2008-04-08 +437 val_437 2008-04-08 +44 val_44 2008-04-08 +444 val_444 2008-04-08 448 val_448 2008-04-08 -194 val_194 2008-04-08 +455 val_455 2008-04-08 +459 val_459 2008-04-08 +459 val_459 2008-04-08 +462 val_462 2008-04-08 +462 val_462 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 477 val_477 2008-04-08 -169 val_169 2008-04-08 -400 val_400 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +484 val_484 2008-04-08 +491 val_491 2008-04-08 +495 val_495 2008-04-08 +51 val_51 2008-04-08 +51 val_51 2008-04-08 +66 val_66 2008-04-08 +77 val_77 2008-04-08 +8 val_8 2008-04-08 +80 val_80 2008-04-08 +84 val_84 2008-04-08 +84 val_84 2008-04-08 +95 val_95 2008-04-08 +95 val_95 2008-04-08 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -415,6 +430,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -426,16 +444,28 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -482,124 +512,124 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### -484 val_484 2008-04-08 0 -224 val_224 2008-04-08 0 -213 val_213 2008-04-08 0 -495 val_495 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -466 val_466 2008-04-08 0 -309 val_309 2008-04-08 0 0 val_0 2008-04-08 0 -316 val_316 2008-04-08 0 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 1 +11 val_11 2008-04-08 1 +114 val_114 2008-04-08 1 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 1 129 val_129 2008-04-08 0 -4 val_4 2008-04-08 0 +129 val_129 2008-04-08 1 +136 val_136 2008-04-08 1 +143 val_143 2008-04-08 0 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +150 val_150 2008-04-08 1 +158 val_158 2008-04-08 0 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 169 val_169 2008-04-08 0 -437 val_437 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +183 val_183 2008-04-08 1 187 val_187 2008-04-08 0 -459 val_459 2008-04-08 0 -103 val_103 2008-04-08 0 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +19 val_19 2008-04-08 1 +190 val_190 2008-04-08 0 +194 val_194 2008-04-08 1 +202 val_202 2008-04-08 1 213 val_213 2008-04-08 0 -275 val_275 2008-04-08 0 -404 val_404 2008-04-08 0 -84 val_84 2008-04-08 0 -8 val_8 2008-04-08 0 -172 val_172 2008-04-08 0 -158 val_158 2008-04-08 0 -26 val_26 2008-04-08 0 -327 val_327 2008-04-08 0 -404 val_404 2008-04-08 0 -282 val_282 2008-04-08 0 -316 val_316 2008-04-08 0 -77 val_77 2008-04-08 0 -118 val_118 2008-04-08 0 -419 val_419 2008-04-08 0 -118 val_118 2008-04-08 0 +213 val_213 2008-04-08 0 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 224 val_224 2008-04-08 0 -389 val_389 2008-04-08 0 +224 val_224 2008-04-08 0 +228 val_228 2008-04-08 1 +235 val_235 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 242 val_242 2008-04-08 0 242 val_242 2008-04-08 0 -95 val_95 2008-04-08 0 -143 val_143 2008-04-08 0 -33 val_33 2008-04-08 0 -367 val_367 2008-04-08 0 -480 val_480 2008-04-08 0 -316 val_316 2008-04-08 0 -80 val_80 2008-04-08 0 -466 val_466 2008-04-08 0 -190 val_190 2008-04-08 0 -396 val_396 2008-04-08 0 -125 val_125 2008-04-08 0 -480 val_480 2008-04-08 0 -305 val_305 2008-04-08 0 -169 val_169 2008-04-08 0 -480 val_480 2008-04-08 0 -172 val_172 2008-04-08 0 +257 val_257 2008-04-08 1 +26 val_26 2008-04-08 0 26 val_26 2008-04-08 0 -341 val_341 2008-04-08 0 -84 val_84 2008-04-08 0 -448 val_448 2008-04-08 0 -477 val_477 2008-04-08 0 -400 val_400 2008-04-08 0 -165 val_165 2008-04-08 1 -150 val_150 2008-04-08 1 -66 val_66 2008-04-08 1 -374 val_374 2008-04-08 1 -37 val_37 2008-04-08 1 -15 val_15 2008-04-08 1 -459 val_459 2008-04-08 1 -396 val_396 2008-04-08 1 -367 val_367 2008-04-08 1 -455 val_455 2008-04-08 1 -345 val_345 2008-04-08 1 -378 val_378 2008-04-08 1 -356 val_356 2008-04-08 1 -125 val_125 2008-04-08 1 -286 val_286 2008-04-08 1 -176 val_176 2008-04-08 1 -51 val_51 2008-04-08 1 -239 val_239 2008-04-08 1 -176 val_176 2008-04-08 1 260 val_260 2008-04-08 1 -217 val_217 2008-04-08 1 -466 val_466 2008-04-08 1 -411 val_411 2008-04-08 1 -129 val_129 2008-04-08 1 -0 val_0 2008-04-08 1 -165 val_165 2008-04-08 1 -51 val_51 2008-04-08 1 -95 val_95 2008-04-08 1 -187 val_187 2008-04-08 1 -169 val_169 2008-04-08 1 -0 val_0 2008-04-08 1 +275 val_275 2008-04-08 0 +282 val_282 2008-04-08 0 282 val_282 2008-04-08 1 -15 val_15 2008-04-08 1 -19 val_19 2008-04-08 1 +286 val_286 2008-04-08 1 +305 val_305 2008-04-08 0 +309 val_309 2008-04-08 0 309 val_309 2008-04-08 1 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +323 val_323 2008-04-08 1 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 327 val_327 2008-04-08 1 +33 val_33 2008-04-08 0 +338 val_338 2008-04-08 0 +341 val_341 2008-04-08 0 +345 val_345 2008-04-08 1 +356 val_356 2008-04-08 1 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +374 val_374 2008-04-08 1 +378 val_378 2008-04-08 1 +389 val_389 2008-04-08 0 392 val_392 2008-04-08 1 +396 val_396 2008-04-08 0 396 val_396 2008-04-08 1 -11 val_11 2008-04-08 1 -228 val_228 2008-04-08 1 -103 val_103 2008-04-08 1 -239 val_239 2008-04-08 1 -202 val_202 2008-04-08 1 -235 val_235 2008-04-08 1 +396 val_396 2008-04-08 1 +4 val_4 2008-04-08 0 +400 val_400 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +411 val_411 2008-04-08 1 +419 val_419 2008-04-08 0 +437 val_437 2008-04-08 0 44 val_44 2008-04-08 1 -257 val_257 2008-04-08 1 -114 val_114 2008-04-08 1 -217 val_217 2008-04-08 1 -187 val_187 2008-04-08 1 -491 val_491 2008-04-08 1 444 val_444 2008-04-08 1 -323 val_323 2008-04-08 1 -136 val_136 2008-04-08 1 +448 val_448 2008-04-08 0 +455 val_455 2008-04-08 1 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 1 462 val_462 2008-04-08 1 462 val_462 2008-04-08 1 -183 val_183 2008-04-08 1 -37 val_37 2008-04-08 1 -194 val_194 2008-04-08 1 -169 val_169 2008-04-08 1 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 1 +477 val_477 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +484 val_484 2008-04-08 0 +491 val_491 2008-04-08 1 +495 val_495 2008-04-08 0 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +66 val_66 2008-04-08 1 +77 val_77 2008-04-08 0 +8 val_8 2008-04-08 0 +80 val_80 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -632,6 +662,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -643,16 +676,28 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -699,242 +744,242 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### -484 val_484 2008-04-08 0 -224 val_224 2008-04-08 0 -213 val_213 2008-04-08 0 -495 val_495 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -466 val_466 2008-04-08 0 -309 val_309 2008-04-08 0 0 val_0 2008-04-08 0 -316 val_316 2008-04-08 0 -129 val_129 2008-04-08 0 -4 val_4 2008-04-08 0 -169 val_169 2008-04-08 0 -437 val_437 2008-04-08 0 -187 val_187 2008-04-08 0 -459 val_459 2008-04-08 0 +0 val_0 2008-04-08 0 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 103 val_103 2008-04-08 0 -213 val_213 2008-04-08 0 -275 val_275 2008-04-08 0 -404 val_404 2008-04-08 0 -84 val_84 2008-04-08 0 -8 val_8 2008-04-08 0 -172 val_172 2008-04-08 0 -158 val_158 2008-04-08 0 -26 val_26 2008-04-08 0 -327 val_327 2008-04-08 0 -404 val_404 2008-04-08 0 -282 val_282 2008-04-08 0 -316 val_316 2008-04-08 0 -77 val_77 2008-04-08 0 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 1 +103 val_103 2008-04-08 1 +11 val_11 2008-04-08 1 +11 val_11 2008-04-08 1 +114 val_114 2008-04-08 1 +114 val_114 2008-04-08 1 118 val_118 2008-04-08 0 -419 val_419 2008-04-08 0 118 val_118 2008-04-08 0 -224 val_224 2008-04-08 0 -389 val_389 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -95 val_95 2008-04-08 0 -143 val_143 2008-04-08 0 -33 val_33 2008-04-08 0 -367 val_367 2008-04-08 0 -480 val_480 2008-04-08 0 -316 val_316 2008-04-08 0 -80 val_80 2008-04-08 0 -466 val_466 2008-04-08 0 -190 val_190 2008-04-08 0 -396 val_396 2008-04-08 0 -125 val_125 2008-04-08 0 -480 val_480 2008-04-08 0 -305 val_305 2008-04-08 0 -169 val_169 2008-04-08 0 -480 val_480 2008-04-08 0 -172 val_172 2008-04-08 0 -26 val_26 2008-04-08 0 -341 val_341 2008-04-08 0 -84 val_84 2008-04-08 0 -448 val_448 2008-04-08 0 -477 val_477 2008-04-08 0 -400 val_400 2008-04-08 0 -484 val_484 2008-04-08 0 -224 val_224 2008-04-08 0 -213 val_213 2008-04-08 0 -495 val_495 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -466 val_466 2008-04-08 0 -309 val_309 2008-04-08 0 -0 val_0 2008-04-08 0 -316 val_316 2008-04-08 0 -129 val_129 2008-04-08 0 -4 val_4 2008-04-08 0 -169 val_169 2008-04-08 0 -437 val_437 2008-04-08 0 -187 val_187 2008-04-08 0 -459 val_459 2008-04-08 0 -103 val_103 2008-04-08 0 -213 val_213 2008-04-08 0 -275 val_275 2008-04-08 0 -404 val_404 2008-04-08 0 -84 val_84 2008-04-08 0 -8 val_8 2008-04-08 0 -172 val_172 2008-04-08 0 -158 val_158 2008-04-08 0 -26 val_26 2008-04-08 0 -327 val_327 2008-04-08 0 -404 val_404 2008-04-08 0 -282 val_282 2008-04-08 0 -316 val_316 2008-04-08 0 -77 val_77 2008-04-08 0 118 val_118 2008-04-08 0 -419 val_419 2008-04-08 0 118 val_118 2008-04-08 0 -224 val_224 2008-04-08 0 -389 val_389 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -95 val_95 2008-04-08 0 -143 val_143 2008-04-08 0 -33 val_33 2008-04-08 0 -367 val_367 2008-04-08 0 -480 val_480 2008-04-08 0 -316 val_316 2008-04-08 0 -80 val_80 2008-04-08 0 -466 val_466 2008-04-08 0 -190 val_190 2008-04-08 0 -396 val_396 2008-04-08 0 125 val_125 2008-04-08 0 -480 val_480 2008-04-08 0 -305 val_305 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 1 +125 val_125 2008-04-08 1 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 1 +129 val_129 2008-04-08 1 +136 val_136 2008-04-08 1 +136 val_136 2008-04-08 1 +143 val_143 2008-04-08 0 +143 val_143 2008-04-08 0 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +150 val_150 2008-04-08 1 +150 val_150 2008-04-08 1 +158 val_158 2008-04-08 0 +158 val_158 2008-04-08 0 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 169 val_169 2008-04-08 0 -480 val_480 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 172 val_172 2008-04-08 0 -26 val_26 2008-04-08 0 -341 val_341 2008-04-08 0 -84 val_84 2008-04-08 0 -448 val_448 2008-04-08 0 -477 val_477 2008-04-08 0 -400 val_400 2008-04-08 0 -165 val_165 2008-04-08 1 -150 val_150 2008-04-08 1 -66 val_66 2008-04-08 1 -374 val_374 2008-04-08 1 -37 val_37 2008-04-08 1 -15 val_15 2008-04-08 1 -459 val_459 2008-04-08 1 -396 val_396 2008-04-08 1 -367 val_367 2008-04-08 1 -455 val_455 2008-04-08 1 -345 val_345 2008-04-08 1 -378 val_378 2008-04-08 1 -356 val_356 2008-04-08 1 -125 val_125 2008-04-08 1 -286 val_286 2008-04-08 1 176 val_176 2008-04-08 1 -51 val_51 2008-04-08 1 -239 val_239 2008-04-08 1 176 val_176 2008-04-08 1 -260 val_260 2008-04-08 1 -217 val_217 2008-04-08 1 -466 val_466 2008-04-08 1 -411 val_411 2008-04-08 1 -129 val_129 2008-04-08 1 -0 val_0 2008-04-08 1 -165 val_165 2008-04-08 1 -51 val_51 2008-04-08 1 -95 val_95 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +183 val_183 2008-04-08 1 +183 val_183 2008-04-08 1 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 187 val_187 2008-04-08 1 -169 val_169 2008-04-08 1 -0 val_0 2008-04-08 1 -282 val_282 2008-04-08 1 -15 val_15 2008-04-08 1 19 val_19 2008-04-08 1 -309 val_309 2008-04-08 1 -327 val_327 2008-04-08 1 -392 val_392 2008-04-08 1 -396 val_396 2008-04-08 1 -11 val_11 2008-04-08 1 -228 val_228 2008-04-08 1 -103 val_103 2008-04-08 1 -239 val_239 2008-04-08 1 +19 val_19 2008-04-08 1 +190 val_190 2008-04-08 0 +190 val_190 2008-04-08 0 +194 val_194 2008-04-08 1 +194 val_194 2008-04-08 1 202 val_202 2008-04-08 1 -235 val_235 2008-04-08 1 -44 val_44 2008-04-08 1 -257 val_257 2008-04-08 1 -114 val_114 2008-04-08 1 +202 val_202 2008-04-08 1 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 217 val_217 2008-04-08 1 -187 val_187 2008-04-08 1 -491 val_491 2008-04-08 1 -444 val_444 2008-04-08 1 -323 val_323 2008-04-08 1 -136 val_136 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -183 val_183 2008-04-08 1 -37 val_37 2008-04-08 1 -194 val_194 2008-04-08 1 -169 val_169 2008-04-08 1 -165 val_165 2008-04-08 1 -150 val_150 2008-04-08 1 -66 val_66 2008-04-08 1 -374 val_374 2008-04-08 1 -37 val_37 2008-04-08 1 -15 val_15 2008-04-08 1 -459 val_459 2008-04-08 1 -396 val_396 2008-04-08 1 -367 val_367 2008-04-08 1 -455 val_455 2008-04-08 1 -345 val_345 2008-04-08 1 -378 val_378 2008-04-08 1 -356 val_356 2008-04-08 1 -125 val_125 2008-04-08 1 -286 val_286 2008-04-08 1 -176 val_176 2008-04-08 1 -51 val_51 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +228 val_228 2008-04-08 1 +228 val_228 2008-04-08 1 +235 val_235 2008-04-08 1 +235 val_235 2008-04-08 1 239 val_239 2008-04-08 1 -176 val_176 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +257 val_257 2008-04-08 1 +257 val_257 2008-04-08 1 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 260 val_260 2008-04-08 1 -217 val_217 2008-04-08 1 -466 val_466 2008-04-08 1 -411 val_411 2008-04-08 1 -129 val_129 2008-04-08 1 -0 val_0 2008-04-08 1 -165 val_165 2008-04-08 1 -51 val_51 2008-04-08 1 -95 val_95 2008-04-08 1 -187 val_187 2008-04-08 1 -169 val_169 2008-04-08 1 -0 val_0 2008-04-08 1 +260 val_260 2008-04-08 1 +275 val_275 2008-04-08 0 +275 val_275 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 0 282 val_282 2008-04-08 1 -15 val_15 2008-04-08 1 -19 val_19 2008-04-08 1 +282 val_282 2008-04-08 1 +286 val_286 2008-04-08 1 +286 val_286 2008-04-08 1 +305 val_305 2008-04-08 0 +305 val_305 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 1 309 val_309 2008-04-08 1 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +323 val_323 2008-04-08 1 +323 val_323 2008-04-08 1 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 1 327 val_327 2008-04-08 1 +33 val_33 2008-04-08 0 +33 val_33 2008-04-08 0 +338 val_338 2008-04-08 0 +338 val_338 2008-04-08 0 +341 val_341 2008-04-08 0 +341 val_341 2008-04-08 0 +345 val_345 2008-04-08 1 +345 val_345 2008-04-08 1 +356 val_356 2008-04-08 1 +356 val_356 2008-04-08 1 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 1 +367 val_367 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +374 val_374 2008-04-08 1 +374 val_374 2008-04-08 1 +378 val_378 2008-04-08 1 +378 val_378 2008-04-08 1 +389 val_389 2008-04-08 0 +389 val_389 2008-04-08 0 392 val_392 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 0 396 val_396 2008-04-08 1 -11 val_11 2008-04-08 1 -228 val_228 2008-04-08 1 -103 val_103 2008-04-08 1 -239 val_239 2008-04-08 1 -202 val_202 2008-04-08 1 -235 val_235 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +4 val_4 2008-04-08 0 +4 val_4 2008-04-08 0 +400 val_400 2008-04-08 0 +400 val_400 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +411 val_411 2008-04-08 1 +411 val_411 2008-04-08 1 +419 val_419 2008-04-08 0 +419 val_419 2008-04-08 0 +437 val_437 2008-04-08 0 +437 val_437 2008-04-08 0 +44 val_44 2008-04-08 1 44 val_44 2008-04-08 1 -257 val_257 2008-04-08 1 -114 val_114 2008-04-08 1 -217 val_217 2008-04-08 1 -187 val_187 2008-04-08 1 -491 val_491 2008-04-08 1 444 val_444 2008-04-08 1 -323 val_323 2008-04-08 1 -136 val_136 2008-04-08 1 +444 val_444 2008-04-08 1 +448 val_448 2008-04-08 0 +448 val_448 2008-04-08 0 +455 val_455 2008-04-08 1 +455 val_455 2008-04-08 1 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 1 +459 val_459 2008-04-08 1 462 val_462 2008-04-08 1 462 val_462 2008-04-08 1 -183 val_183 2008-04-08 1 -37 val_37 2008-04-08 1 -194 val_194 2008-04-08 1 -169 val_169 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 1 +466 val_466 2008-04-08 1 +477 val_477 2008-04-08 0 +477 val_477 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +484 val_484 2008-04-08 0 +484 val_484 2008-04-08 0 +491 val_491 2008-04-08 1 +491 val_491 2008-04-08 1 +495 val_495 2008-04-08 0 +495 val_495 2008-04-08 0 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +66 val_66 2008-04-08 1 +66 val_66 2008-04-08 1 +77 val_77 2008-04-08 0 +77 val_77 2008-04-08 0 +8 val_8 2008-04-08 0 +8 val_8 2008-04-08 0 +80 val_80 2008-04-08 0 +80 val_80 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 1 +95 val_95 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -1041,124 +1086,124 @@ POSTHOOK: query: select * from srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8 #### A masked pattern was here #### -8 val_8 -33 val_33 -37 val_37 -37 val_37 -51 val_51 -51 val_51 -103 val_103 -103 val_103 -125 val_125 -125 val_125 -165 val_165 -165 val_165 -194 val_194 -260 val_260 -275 val_275 -305 val_305 -323 val_323 -411 val_411 -455 val_455 -466 val_466 -466 val_466 -466 val_466 -484 val_484 0 val_0 0 val_0 0 val_0 -4 val_4 +103 val_103 +103 val_103 11 val_11 -19 val_19 -26 val_26 -26 val_26 -44 val_44 -77 val_77 +114 val_114 118 val_118 118 val_118 -158 val_158 -217 val_217 -217 val_217 -316 val_316 -316 val_316 -316 val_316 -327 val_327 -327 val_327 -327 val_327 -338 val_338 -356 val_356 -378 val_378 -389 val_389 -404 val_404 -404 val_404 -419 val_419 -444 val_444 -462 val_462 -462 val_462 -15 val_15 -15 val_15 -80 val_80 -84 val_84 -84 val_84 -95 val_95 -95 val_95 +125 val_125 +125 val_125 129 val_129 129 val_129 +136 val_136 +143 val_143 +15 val_15 +15 val_15 +150 val_150 +158 val_158 +165 val_165 +165 val_165 169 val_169 169 val_169 169 val_169 169 val_169 -239 val_239 -239 val_239 -282 val_282 -282 val_282 -367 val_367 -367 val_367 -66 val_66 -114 val_114 -136 val_136 -143 val_143 172 val_172 172 val_172 -190 val_190 -242 val_242 -242 val_242 -286 val_286 -345 val_345 -400 val_400 -437 val_437 -448 val_448 -459 val_459 -459 val_459 -150 val_150 176 val_176 176 val_176 183 val_183 187 val_187 187 val_187 187 val_187 +19 val_19 +190 val_190 +194 val_194 202 val_202 213 val_213 213 val_213 +217 val_217 +217 val_217 224 val_224 224 val_224 228 val_228 235 val_235 +239 val_239 +239 val_239 +242 val_242 +242 val_242 257 val_257 +26 val_26 +26 val_26 +260 val_260 +275 val_275 +282 val_282 +282 val_282 +286 val_286 +305 val_305 309 val_309 309 val_309 +316 val_316 +316 val_316 +316 val_316 +323 val_323 +327 val_327 +327 val_327 +327 val_327 +33 val_33 +338 val_338 341 val_341 +345 val_345 +356 val_356 +367 val_367 +367 val_367 +37 val_37 +37 val_37 374 val_374 +378 val_378 +389 val_389 392 val_392 396 val_396 396 val_396 396 val_396 +4 val_4 +400 val_400 +404 val_404 +404 val_404 +411 val_411 +419 val_419 +437 val_437 +44 val_44 +444 val_444 +448 val_448 +455 val_455 +459 val_459 +459 val_459 +462 val_462 +462 val_462 +466 val_466 +466 val_466 +466 val_466 477 val_477 480 val_480 480 val_480 480 val_480 +484 val_484 491 val_491 495 val_495 +51 val_51 +51 val_51 +66 val_66 +77 val_77 +8 val_8 +80 val_80 +84 val_84 +84 val_84 +95 val_95 +95 val_95 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -1206,250 +1251,22 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string) - Execution mode: vectorized, llap - LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 - - Stage: Stage-2 - Dependency Collection - - Stage: Stage-0 - Move Operator - tables: - partition: - ds - replace: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 - - Stage: Stage-3 - Stats Work - Basic Stats Work: - -PREHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8 -PREHOOK: type: QUERY -PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ -PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ -POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08 -POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: select * from srcbucket_mapjoin_n8 -PREHOOK: type: QUERY -PREHOOK: Input: default@srcbucket_mapjoin_n8 -PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 -#### A masked pattern was here #### -POSTHOOK: query: select * from srcbucket_mapjoin_n8 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcbucket_mapjoin_n8 -POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 -#### A masked pattern was here #### -8 val_8 2008-04-08 -33 val_33 2008-04-08 -37 val_37 2008-04-08 -37 val_37 2008-04-08 -51 val_51 2008-04-08 -51 val_51 2008-04-08 -103 val_103 2008-04-08 -103 val_103 2008-04-08 -125 val_125 2008-04-08 -125 val_125 2008-04-08 -165 val_165 2008-04-08 -165 val_165 2008-04-08 -194 val_194 2008-04-08 -260 val_260 2008-04-08 -275 val_275 2008-04-08 -305 val_305 2008-04-08 -323 val_323 2008-04-08 -411 val_411 2008-04-08 -455 val_455 2008-04-08 -466 val_466 2008-04-08 -466 val_466 2008-04-08 -466 val_466 2008-04-08 -484 val_484 2008-04-08 -0 val_0 2008-04-08 -0 val_0 2008-04-08 -0 val_0 2008-04-08 -4 val_4 2008-04-08 -11 val_11 2008-04-08 -19 val_19 2008-04-08 -26 val_26 2008-04-08 -26 val_26 2008-04-08 -44 val_44 2008-04-08 -77 val_77 2008-04-08 -118 val_118 2008-04-08 -118 val_118 2008-04-08 -158 val_158 2008-04-08 -217 val_217 2008-04-08 -217 val_217 2008-04-08 -316 val_316 2008-04-08 -316 val_316 2008-04-08 -316 val_316 2008-04-08 -327 val_327 2008-04-08 -327 val_327 2008-04-08 -327 val_327 2008-04-08 -338 val_338 2008-04-08 -356 val_356 2008-04-08 -378 val_378 2008-04-08 -389 val_389 2008-04-08 -404 val_404 2008-04-08 -404 val_404 2008-04-08 -419 val_419 2008-04-08 -444 val_444 2008-04-08 -462 val_462 2008-04-08 -462 val_462 2008-04-08 -15 val_15 2008-04-08 -15 val_15 2008-04-08 -80 val_80 2008-04-08 -84 val_84 2008-04-08 -84 val_84 2008-04-08 -95 val_95 2008-04-08 -95 val_95 2008-04-08 -129 val_129 2008-04-08 -129 val_129 2008-04-08 -169 val_169 2008-04-08 -169 val_169 2008-04-08 -169 val_169 2008-04-08 -169 val_169 2008-04-08 -239 val_239 2008-04-08 -239 val_239 2008-04-08 -282 val_282 2008-04-08 -282 val_282 2008-04-08 -367 val_367 2008-04-08 -367 val_367 2008-04-08 -66 val_66 2008-04-08 -114 val_114 2008-04-08 -136 val_136 2008-04-08 -143 val_143 2008-04-08 -172 val_172 2008-04-08 -172 val_172 2008-04-08 -190 val_190 2008-04-08 -242 val_242 2008-04-08 -242 val_242 2008-04-08 -286 val_286 2008-04-08 -345 val_345 2008-04-08 -400 val_400 2008-04-08 -437 val_437 2008-04-08 -448 val_448 2008-04-08 -459 val_459 2008-04-08 -459 val_459 2008-04-08 -150 val_150 2008-04-08 -176 val_176 2008-04-08 -176 val_176 2008-04-08 -183 val_183 2008-04-08 -187 val_187 2008-04-08 -187 val_187 2008-04-08 -187 val_187 2008-04-08 -202 val_202 2008-04-08 -213 val_213 2008-04-08 -213 val_213 2008-04-08 -224 val_224 2008-04-08 -224 val_224 2008-04-08 -228 val_228 2008-04-08 -235 val_235 2008-04-08 -257 val_257 2008-04-08 -309 val_309 2008-04-08 -309 val_309 2008-04-08 -341 val_341 2008-04-08 -374 val_374 2008-04-08 -392 val_392 2008-04-08 -396 val_396 2008-04-08 -396 val_396 2008-04-08 -396 val_396 2008-04-08 -477 val_477 2008-04-08 -480 val_480 2008-04-08 -480 val_480 2008-04-08 -480 val_480 2008-04-08 -491 val_491 2008-04-08 -495 val_495 2008-04-08 -PREHOOK: query: drop table srcbucket_mapjoin_n8 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@srcbucket_mapjoin_n8 -PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: drop table srcbucket_mapjoin_n8 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@srcbucket_mapjoin_n8 -POSTHOOK: Output: default@srcbucket_mapjoin_n8 -PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@srcbucket_mapjoin_n8 -PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 -PREHOOK: type: QUERY -PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ -PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: srcbucket_mapjoin_n8__temp_table_for_load_data__ - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + key expressions: _col2 (type: string), _bucket_number (type: string), _col0 (type: int) + sort order: +++ + Map-reduce partition columns: _col2 (type: string) + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _bucket_number File Output Operator compressed: false - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1464,7 +1281,6 @@ STAGE PLANS: tables: partition: ds - hr replace: false table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -1476,149 +1292,144 @@ STAGE PLANS: Stats Work Basic Stats Work: -PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/load_data_1_partition.txt' INTO TABLE srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ -POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 -POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 -POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] -POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] -POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08 +POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] PREHOOK: query: select * from srcbucket_mapjoin_n8 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_n8 -PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 -PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 +PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 #### A masked pattern was here #### POSTHOOK: query: select * from srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8 -POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 -POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 +POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 #### A masked pattern was here #### -8 val_8 2008-04-08 0 -33 val_33 2008-04-08 0 -103 val_103 2008-04-08 0 -125 val_125 2008-04-08 0 -275 val_275 2008-04-08 0 -305 val_305 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -484 val_484 2008-04-08 0 -0 val_0 2008-04-08 0 -4 val_4 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -77 val_77 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -158 val_158 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -389 val_389 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -419 val_419 2008-04-08 0 -80 val_80 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -95 val_95 2008-04-08 0 -129 val_129 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -282 val_282 2008-04-08 0 -367 val_367 2008-04-08 0 -143 val_143 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -190 val_190 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -400 val_400 2008-04-08 0 -437 val_437 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -187 val_187 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -309 val_309 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -477 val_477 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -495 val_495 2008-04-08 0 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -103 val_103 2008-04-08 1 -125 val_125 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 -323 val_323 2008-04-08 1 -411 val_411 2008-04-08 1 -455 val_455 2008-04-08 1 -466 val_466 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -11 val_11 2008-04-08 1 -19 val_19 2008-04-08 1 -44 val_44 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -327 val_327 2008-04-08 1 -356 val_356 2008-04-08 1 -378 val_378 2008-04-08 1 -444 val_444 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -95 val_95 2008-04-08 1 -129 val_129 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -282 val_282 2008-04-08 1 -367 val_367 2008-04-08 1 -66 val_66 2008-04-08 1 -114 val_114 2008-04-08 1 -136 val_136 2008-04-08 1 -286 val_286 2008-04-08 1 -345 val_345 2008-04-08 1 -459 val_459 2008-04-08 1 -150 val_150 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 -183 val_183 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -202 val_202 2008-04-08 1 -228 val_228 2008-04-08 1 -235 val_235 2008-04-08 1 -257 val_257 2008-04-08 1 -309 val_309 2008-04-08 1 -374 val_374 2008-04-08 1 -392 val_392 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -491 val_491 2008-04-08 1 +0 val_0 2008-04-08 +0 val_0 2008-04-08 +0 val_0 2008-04-08 +103 val_103 2008-04-08 +103 val_103 2008-04-08 +11 val_11 2008-04-08 +114 val_114 2008-04-08 +118 val_118 2008-04-08 +118 val_118 2008-04-08 +125 val_125 2008-04-08 +125 val_125 2008-04-08 +129 val_129 2008-04-08 +129 val_129 2008-04-08 +136 val_136 2008-04-08 +143 val_143 2008-04-08 +15 val_15 2008-04-08 +15 val_15 2008-04-08 +150 val_150 2008-04-08 +158 val_158 2008-04-08 +165 val_165 2008-04-08 +165 val_165 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +172 val_172 2008-04-08 +172 val_172 2008-04-08 +176 val_176 2008-04-08 +176 val_176 2008-04-08 +183 val_183 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +19 val_19 2008-04-08 +190 val_190 2008-04-08 +194 val_194 2008-04-08 +202 val_202 2008-04-08 +213 val_213 2008-04-08 +213 val_213 2008-04-08 +217 val_217 2008-04-08 +217 val_217 2008-04-08 +224 val_224 2008-04-08 +224 val_224 2008-04-08 +228 val_228 2008-04-08 +235 val_235 2008-04-08 +239 val_239 2008-04-08 +239 val_239 2008-04-08 +242 val_242 2008-04-08 +242 val_242 2008-04-08 +257 val_257 2008-04-08 +26 val_26 2008-04-08 +26 val_26 2008-04-08 +260 val_260 2008-04-08 +275 val_275 2008-04-08 +282 val_282 2008-04-08 +282 val_282 2008-04-08 +286 val_286 2008-04-08 +305 val_305 2008-04-08 +309 val_309 2008-04-08 +309 val_309 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +323 val_323 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 +33 val_33 2008-04-08 +338 val_338 2008-04-08 +341 val_341 2008-04-08 +345 val_345 2008-04-08 +356 val_356 2008-04-08 +367 val_367 2008-04-08 +367 val_367 2008-04-08 +37 val_37 2008-04-08 +37 val_37 2008-04-08 +374 val_374 2008-04-08 +378 val_378 2008-04-08 +389 val_389 2008-04-08 +392 val_392 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +4 val_4 2008-04-08 +400 val_400 2008-04-08 +404 val_404 2008-04-08 +404 val_404 2008-04-08 +411 val_411 2008-04-08 +419 val_419 2008-04-08 +437 val_437 2008-04-08 +44 val_44 2008-04-08 +444 val_444 2008-04-08 +448 val_448 2008-04-08 +455 val_455 2008-04-08 +459 val_459 2008-04-08 +459 val_459 2008-04-08 +462 val_462 2008-04-08 +462 val_462 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +477 val_477 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +484 val_484 2008-04-08 +491 val_491 2008-04-08 +495 val_495 2008-04-08 +51 val_51 2008-04-08 +51 val_51 2008-04-08 +66 val_66 2008-04-08 +77 val_77 2008-04-08 +8 val_8 2008-04-08 +80 val_80 2008-04-08 +84 val_84 2008-04-08 +84 val_84 2008-04-08 +95 val_95 2008-04-08 +95 val_95 2008-04-08 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -1635,11 +1446,11 @@ POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partit POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@srcbucket_mapjoin_n8 -PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ STAGE DEPENDENCIES: @@ -1660,29 +1471,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin_n8__temp_table_for_load_data__ - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: int), _bucket_number (type: string), _col0 (type: int) + sort order: ++++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: int) + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number File Output Operator compressed: false - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -1709,11 +1519,11 @@ STAGE PLANS: Stats Work Basic Stats Work: -PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/load_data_2_partitions.txt' INTO TABLE srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 @@ -1734,242 +1544,124 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 0 val_0 2008-04-08 0 -0 val_0 2008-04-08 0 -4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -77 val_77 2008-04-08 0 -77 val_77 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 1 +11 val_11 2008-04-08 1 +114 val_114 2008-04-08 1 118 val_118 2008-04-08 0 118 val_118 2008-04-08 0 -158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -338 val_338 2008-04-08 0 -389 val_389 2008-04-08 0 -389 val_389 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -95 val_95 2008-04-08 0 -95 val_95 2008-04-08 0 -129 val_129 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 1 129 val_129 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -143 val_143 2008-04-08 0 +129 val_129 2008-04-08 1 +136 val_136 2008-04-08 1 143 val_143 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -190 val_190 2008-04-08 0 -190 val_190 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -400 val_400 2008-04-08 0 -400 val_400 2008-04-08 0 -437 val_437 2008-04-08 0 -437 val_437 2008-04-08 0 -448 val_448 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -187 val_187 2008-04-08 0 -187 val_187 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -477 val_477 2008-04-08 0 -477 val_477 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -495 val_495 2008-04-08 0 -495 val_495 2008-04-08 0 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 -466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -11 val_11 2008-04-08 1 -11 val_11 2008-04-08 1 -19 val_19 2008-04-08 1 -19 val_19 2008-04-08 1 -44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -444 val_444 2008-04-08 1 -444 val_444 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 15 val_15 2008-04-08 1 15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 +150 val_150 2008-04-08 1 +158 val_158 2008-04-08 0 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 169 val_169 2008-04-08 1 169 val_169 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -66 val_66 2008-04-08 1 -66 val_66 2008-04-08 1 -114 val_114 2008-04-08 1 -114 val_114 2008-04-08 1 -136 val_136 2008-04-08 1 -136 val_136 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -459 val_459 2008-04-08 1 -459 val_459 2008-04-08 1 -150 val_150 2008-04-08 1 -150 val_150 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 176 val_176 2008-04-08 1 176 val_176 2008-04-08 1 183 val_183 2008-04-08 1 -183 val_183 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 +187 val_187 2008-04-08 0 187 val_187 2008-04-08 1 187 val_187 2008-04-08 1 +19 val_19 2008-04-08 1 +190 val_190 2008-04-08 0 +194 val_194 2008-04-08 1 202 val_202 2008-04-08 1 -202 val_202 2008-04-08 1 -228 val_228 2008-04-08 1 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 228 val_228 2008-04-08 1 235 val_235 2008-04-08 1 -235 val_235 2008-04-08 1 -257 val_257 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 257 val_257 2008-04-08 1 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +260 val_260 2008-04-08 1 +275 val_275 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 1 +286 val_286 2008-04-08 1 +305 val_305 2008-04-08 0 +309 val_309 2008-04-08 0 309 val_309 2008-04-08 1 -309 val_309 2008-04-08 1 -374 val_374 2008-04-08 1 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +323 val_323 2008-04-08 1 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 1 +33 val_33 2008-04-08 0 +338 val_338 2008-04-08 0 +341 val_341 2008-04-08 0 +345 val_345 2008-04-08 1 +356 val_356 2008-04-08 1 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 374 val_374 2008-04-08 1 +378 val_378 2008-04-08 1 +389 val_389 2008-04-08 0 392 val_392 2008-04-08 1 -392 val_392 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 +396 val_396 2008-04-08 0 396 val_396 2008-04-08 1 396 val_396 2008-04-08 1 +4 val_4 2008-04-08 0 +400 val_400 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +411 val_411 2008-04-08 1 +419 val_419 2008-04-08 0 +437 val_437 2008-04-08 0 +44 val_44 2008-04-08 1 +444 val_444 2008-04-08 1 +448 val_448 2008-04-08 0 +455 val_455 2008-04-08 1 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 1 +477 val_477 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +484 val_484 2008-04-08 0 491 val_491 2008-04-08 1 -491 val_491 2008-04-08 1 +495 val_495 2008-04-08 0 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +66 val_66 2008-04-08 1 +77 val_77 2008-04-08 0 +8 val_8 2008-04-08 0 +80 val_80 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -1986,11 +1678,11 @@ POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partit POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@srcbucket_mapjoin_n8 -PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ STAGE DEPENDENCIES: @@ -2011,29 +1703,28 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcbucket_mapjoin_n8__temp_table_for_load_data__ - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + key expressions: _col2 (type: string), _col3 (type: int), _bucket_number (type: string), _col0 (type: int) + sort order: ++++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: int) + value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number File Output Operator compressed: false - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2060,11 +1751,11 @@ STAGE PLANS: Stats Work Basic Stats Work: -PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 PREHOOK: type: QUERY PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ PREHOOK: Output: default@srcbucket_mapjoin_n8 -POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions/subdir' INTO TABLE srcbucket_mapjoin_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 @@ -2085,224 +1776,208 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 -0 val_0 2008-04-08 0 0 val_0 2008-04-08 0 0 val_0 2008-04-08 0 -0 val_0 2008-04-08 0 -4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -77 val_77 2008-04-08 0 -77 val_77 2008-04-08 0 -77 val_77 2008-04-08 0 -77 val_77 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +0 val_0 2008-04-08 1 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 1 +103 val_103 2008-04-08 1 +11 val_11 2008-04-08 1 +11 val_11 2008-04-08 1 +114 val_114 2008-04-08 1 +114 val_114 2008-04-08 1 118 val_118 2008-04-08 0 118 val_118 2008-04-08 0 118 val_118 2008-04-08 0 118 val_118 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 1 +125 val_125 2008-04-08 1 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 1 +129 val_129 2008-04-08 1 +136 val_136 2008-04-08 1 +136 val_136 2008-04-08 1 +143 val_143 2008-04-08 0 +143 val_143 2008-04-08 0 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +150 val_150 2008-04-08 1 +150 val_150 2008-04-08 1 158 val_158 2008-04-08 0 158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +176 val_176 2008-04-08 1 +183 val_183 2008-04-08 1 +183 val_183 2008-04-08 1 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +19 val_19 2008-04-08 1 +19 val_19 2008-04-08 1 +190 val_190 2008-04-08 0 +190 val_190 2008-04-08 0 +194 val_194 2008-04-08 1 +194 val_194 2008-04-08 1 +202 val_202 2008-04-08 1 +202 val_202 2008-04-08 1 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +228 val_228 2008-04-08 1 +228 val_228 2008-04-08 1 +235 val_235 2008-04-08 1 +235 val_235 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +257 val_257 2008-04-08 1 +257 val_257 2008-04-08 1 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +260 val_260 2008-04-08 1 +260 val_260 2008-04-08 1 +275 val_275 2008-04-08 0 +275 val_275 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 1 +282 val_282 2008-04-08 1 +286 val_286 2008-04-08 1 +286 val_286 2008-04-08 1 +305 val_305 2008-04-08 0 +305 val_305 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 1 +309 val_309 2008-04-08 1 316 val_316 2008-04-08 0 316 val_316 2008-04-08 0 316 val_316 2008-04-08 0 316 val_316 2008-04-08 0 316 val_316 2008-04-08 0 316 val_316 2008-04-08 0 +323 val_323 2008-04-08 1 +323 val_323 2008-04-08 1 327 val_327 2008-04-08 0 327 val_327 2008-04-08 0 327 val_327 2008-04-08 0 327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -338 val_338 2008-04-08 0 +327 val_327 2008-04-08 1 +327 val_327 2008-04-08 1 +33 val_33 2008-04-08 0 +33 val_33 2008-04-08 0 338 val_338 2008-04-08 0 338 val_338 2008-04-08 0 +341 val_341 2008-04-08 0 +341 val_341 2008-04-08 0 +345 val_345 2008-04-08 1 +345 val_345 2008-04-08 1 +356 val_356 2008-04-08 1 +356 val_356 2008-04-08 1 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 1 +367 val_367 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +374 val_374 2008-04-08 1 +374 val_374 2008-04-08 1 +378 val_378 2008-04-08 1 +378 val_378 2008-04-08 1 389 val_389 2008-04-08 0 389 val_389 2008-04-08 0 -389 val_389 2008-04-08 0 -389 val_389 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 +392 val_392 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +396 val_396 2008-04-08 1 +4 val_4 2008-04-08 0 +4 val_4 2008-04-08 0 +400 val_400 2008-04-08 0 +400 val_400 2008-04-08 0 404 val_404 2008-04-08 0 404 val_404 2008-04-08 0 404 val_404 2008-04-08 0 404 val_404 2008-04-08 0 +411 val_411 2008-04-08 1 +411 val_411 2008-04-08 1 419 val_419 2008-04-08 0 419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -95 val_95 2008-04-08 0 -95 val_95 2008-04-08 0 -95 val_95 2008-04-08 0 -95 val_95 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -143 val_143 2008-04-08 0 -143 val_143 2008-04-08 0 -143 val_143 2008-04-08 0 -143 val_143 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -190 val_190 2008-04-08 0 -190 val_190 2008-04-08 0 -190 val_190 2008-04-08 0 -190 val_190 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -400 val_400 2008-04-08 0 -400 val_400 2008-04-08 0 -400 val_400 2008-04-08 0 -400 val_400 2008-04-08 0 -437 val_437 2008-04-08 0 437 val_437 2008-04-08 0 437 val_437 2008-04-08 0 -437 val_437 2008-04-08 0 -448 val_448 2008-04-08 0 -448 val_448 2008-04-08 0 +44 val_44 2008-04-08 1 +44 val_44 2008-04-08 1 +444 val_444 2008-04-08 1 +444 val_444 2008-04-08 1 448 val_448 2008-04-08 0 448 val_448 2008-04-08 0 +455 val_455 2008-04-08 1 +455 val_455 2008-04-08 1 459 val_459 2008-04-08 0 459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -187 val_187 2008-04-08 0 -187 val_187 2008-04-08 0 -187 val_187 2008-04-08 0 -187 val_187 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -477 val_477 2008-04-08 0 -477 val_477 2008-04-08 0 +459 val_459 2008-04-08 1 +459 val_459 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 1 +466 val_466 2008-04-08 1 477 val_477 2008-04-08 0 477 val_477 2008-04-08 0 480 val_480 2008-04-08 0 @@ -2311,72 +1986,150 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 480 val_480 2008-04-08 0 480 val_480 2008-04-08 0 480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -495 val_495 2008-04-08 0 -495 val_495 2008-04-08 0 +484 val_484 2008-04-08 0 +484 val_484 2008-04-08 0 +491 val_491 2008-04-08 1 +491 val_491 2008-04-08 1 495 val_495 2008-04-08 0 495 val_495 2008-04-08 0 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -51 val_51 2008-04-08 1 51 val_51 2008-04-08 1 51 val_51 2008-04-08 1 51 val_51 2008-04-08 1 51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 -466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 +66 val_66 2008-04-08 1 +66 val_66 2008-04-08 1 +77 val_77 2008-04-08 0 +77 val_77 2008-04-08 0 +8 val_8 2008-04-08 0 +8 val_8 2008-04-08 0 +80 val_80 2008-04-08 0 +80 val_80 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 1 +95 val_95 2008-04-08 1 +PREHOOK: query: drop table srcbucket_mapjoin_n8 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcbucket_mapjoin_n8 +PREHOOK: Output: default@srcbucket_mapjoin_n8 +POSTHOOK: query: drop table srcbucket_mapjoin_n8 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcbucket_mapjoin_n8 +POSTHOOK: Output: default@srcbucket_mapjoin_n8 +PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin_n8 +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n8(key int, value string) partitioned by (ds string, hr int) clustered by (key) sorted by (key) into 5 buckets STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin_n8 +PREHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ +PREHOOK: Output: default@srcbucket_mapjoin_n8 +POSTHOOK: query: explain load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcbucket_mapjoin_n8__temp_table_for_load_data__ + Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: string), _col3 (type: int), _bucket_number (type: string), _col0 (type: int) + sort order: ++++ + Map-reduce partition columns: _col2 (type: string), _col3 (type: int) + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number + File Output Operator + compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED + Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds + hr + replace: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ +PREHOOK: Output: default@srcbucket_mapjoin_n8 +POSTHOOK: query: load data local inpath '../../data/files/load_data_job/partitions' INTO TABLE srcbucket_mapjoin_n8 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_n8__temp_table_for_load_data__ +POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 +POSTHOOK: Output: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 +POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=0).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=0).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=1).key SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: srcbucket_mapjoin_n8 PARTITION(ds=2008-04-08,hr=1).value SIMPLE [(srcbucket_mapjoin_n8__temp_table_for_load_data__)srcbucket_mapjoin_n8__temp_table_for_load_data__.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: select * from srcbucket_mapjoin_n8 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_n8 +PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 +PREHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +POSTHOOK: query: select * from srcbucket_mapjoin_n8 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_n8 +POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 +POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 +#### A masked pattern was here #### +0 val_0 2008-04-08 0 +0 val_0 2008-04-08 0 +0 val_0 2008-04-08 0 +0 val_0 2008-04-08 0 0 val_0 2008-04-08 1 0 val_0 2008-04-08 1 0 val_0 2008-04-08 1 @@ -2385,118 +2138,102 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 0 val_0 2008-04-08 1 0 val_0 2008-04-08 1 0 val_0 2008-04-08 1 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 0 +103 val_103 2008-04-08 1 +103 val_103 2008-04-08 1 +103 val_103 2008-04-08 1 +103 val_103 2008-04-08 1 11 val_11 2008-04-08 1 11 val_11 2008-04-08 1 11 val_11 2008-04-08 1 11 val_11 2008-04-08 1 -19 val_19 2008-04-08 1 -19 val_19 2008-04-08 1 -19 val_19 2008-04-08 1 -19 val_19 2008-04-08 1 -44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -444 val_444 2008-04-08 1 -444 val_444 2008-04-08 1 -444 val_444 2008-04-08 1 -444 val_444 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -66 val_66 2008-04-08 1 -66 val_66 2008-04-08 1 -66 val_66 2008-04-08 1 -66 val_66 2008-04-08 1 114 val_114 2008-04-08 1 114 val_114 2008-04-08 1 114 val_114 2008-04-08 1 114 val_114 2008-04-08 1 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 0 +125 val_125 2008-04-08 1 +125 val_125 2008-04-08 1 +125 val_125 2008-04-08 1 +125 val_125 2008-04-08 1 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 1 +129 val_129 2008-04-08 1 +129 val_129 2008-04-08 1 +129 val_129 2008-04-08 1 136 val_136 2008-04-08 1 136 val_136 2008-04-08 1 136 val_136 2008-04-08 1 136 val_136 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -459 val_459 2008-04-08 1 -459 val_459 2008-04-08 1 -459 val_459 2008-04-08 1 -459 val_459 2008-04-08 1 +143 val_143 2008-04-08 0 +143 val_143 2008-04-08 0 +143 val_143 2008-04-08 0 +143 val_143 2008-04-08 0 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 150 val_150 2008-04-08 1 150 val_150 2008-04-08 1 150 val_150 2008-04-08 1 150 val_150 2008-04-08 1 +158 val_158 2008-04-08 0 +158 val_158 2008-04-08 0 +158 val_158 2008-04-08 0 +158 val_158 2008-04-08 0 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +165 val_165 2008-04-08 1 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 176 val_176 2008-04-08 1 176 val_176 2008-04-08 1 176 val_176 2008-04-08 1 @@ -2509,6 +2246,10 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 183 val_183 2008-04-08 1 183 val_183 2008-04-08 1 183 val_183 2008-04-08 1 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 0 187 val_187 2008-04-08 1 187 val_187 2008-04-08 1 187 val_187 2008-04-08 1 @@ -2517,10 +2258,46 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 187 val_187 2008-04-08 1 187 val_187 2008-04-08 1 187 val_187 2008-04-08 1 +19 val_19 2008-04-08 1 +19 val_19 2008-04-08 1 +19 val_19 2008-04-08 1 +19 val_19 2008-04-08 1 +190 val_190 2008-04-08 0 +190 val_190 2008-04-08 0 +190 val_190 2008-04-08 0 +190 val_190 2008-04-08 0 +194 val_194 2008-04-08 1 +194 val_194 2008-04-08 1 +194 val_194 2008-04-08 1 +194 val_194 2008-04-08 1 202 val_202 2008-04-08 1 202 val_202 2008-04-08 1 202 val_202 2008-04-08 1 202 val_202 2008-04-08 1 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +217 val_217 2008-04-08 1 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 228 val_228 2008-04-08 1 228 val_228 2008-04-08 1 228 val_228 2008-04-08 1 @@ -2529,22 +2306,150 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 235 val_235 2008-04-08 1 235 val_235 2008-04-08 1 235 val_235 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +239 val_239 2008-04-08 1 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 257 val_257 2008-04-08 1 257 val_257 2008-04-08 1 257 val_257 2008-04-08 1 257 val_257 2008-04-08 1 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +260 val_260 2008-04-08 1 +260 val_260 2008-04-08 1 +260 val_260 2008-04-08 1 +260 val_260 2008-04-08 1 +275 val_275 2008-04-08 0 +275 val_275 2008-04-08 0 +275 val_275 2008-04-08 0 +275 val_275 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 1 +282 val_282 2008-04-08 1 +282 val_282 2008-04-08 1 +282 val_282 2008-04-08 1 +286 val_286 2008-04-08 1 +286 val_286 2008-04-08 1 +286 val_286 2008-04-08 1 +286 val_286 2008-04-08 1 +305 val_305 2008-04-08 0 +305 val_305 2008-04-08 0 +305 val_305 2008-04-08 0 +305 val_305 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 0 +309 val_309 2008-04-08 0 309 val_309 2008-04-08 1 309 val_309 2008-04-08 1 309 val_309 2008-04-08 1 309 val_309 2008-04-08 1 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +323 val_323 2008-04-08 1 +323 val_323 2008-04-08 1 +323 val_323 2008-04-08 1 +323 val_323 2008-04-08 1 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 1 +327 val_327 2008-04-08 1 +327 val_327 2008-04-08 1 +327 val_327 2008-04-08 1 +33 val_33 2008-04-08 0 +33 val_33 2008-04-08 0 +33 val_33 2008-04-08 0 +33 val_33 2008-04-08 0 +338 val_338 2008-04-08 0 +338 val_338 2008-04-08 0 +338 val_338 2008-04-08 0 +338 val_338 2008-04-08 0 +341 val_341 2008-04-08 0 +341 val_341 2008-04-08 0 +341 val_341 2008-04-08 0 +341 val_341 2008-04-08 0 +345 val_345 2008-04-08 1 +345 val_345 2008-04-08 1 +345 val_345 2008-04-08 1 +345 val_345 2008-04-08 1 +356 val_356 2008-04-08 1 +356 val_356 2008-04-08 1 +356 val_356 2008-04-08 1 +356 val_356 2008-04-08 1 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 1 +367 val_367 2008-04-08 1 +367 val_367 2008-04-08 1 +367 val_367 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 374 val_374 2008-04-08 1 374 val_374 2008-04-08 1 374 val_374 2008-04-08 1 374 val_374 2008-04-08 1 +378 val_378 2008-04-08 1 +378 val_378 2008-04-08 1 +378 val_378 2008-04-08 1 +378 val_378 2008-04-08 1 +389 val_389 2008-04-08 0 +389 val_389 2008-04-08 0 +389 val_389 2008-04-08 0 +389 val_389 2008-04-08 0 392 val_392 2008-04-08 1 392 val_392 2008-04-08 1 392 val_392 2008-04-08 1 392 val_392 2008-04-08 1 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 0 +396 val_396 2008-04-08 0 396 val_396 2008-04-08 1 396 val_396 2008-04-08 1 396 val_396 2008-04-08 1 @@ -2553,10 +2458,146 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 396 val_396 2008-04-08 1 396 val_396 2008-04-08 1 396 val_396 2008-04-08 1 +4 val_4 2008-04-08 0 +4 val_4 2008-04-08 0 +4 val_4 2008-04-08 0 +4 val_4 2008-04-08 0 +400 val_400 2008-04-08 0 +400 val_400 2008-04-08 0 +400 val_400 2008-04-08 0 +400 val_400 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +411 val_411 2008-04-08 1 +411 val_411 2008-04-08 1 +411 val_411 2008-04-08 1 +411 val_411 2008-04-08 1 +419 val_419 2008-04-08 0 +419 val_419 2008-04-08 0 +419 val_419 2008-04-08 0 +419 val_419 2008-04-08 0 +437 val_437 2008-04-08 0 +437 val_437 2008-04-08 0 +437 val_437 2008-04-08 0 +437 val_437 2008-04-08 0 +44 val_44 2008-04-08 1 +44 val_44 2008-04-08 1 +44 val_44 2008-04-08 1 +44 val_44 2008-04-08 1 +444 val_444 2008-04-08 1 +444 val_444 2008-04-08 1 +444 val_444 2008-04-08 1 +444 val_444 2008-04-08 1 +448 val_448 2008-04-08 0 +448 val_448 2008-04-08 0 +448 val_448 2008-04-08 0 +448 val_448 2008-04-08 0 +455 val_455 2008-04-08 1 +455 val_455 2008-04-08 1 +455 val_455 2008-04-08 1 +455 val_455 2008-04-08 1 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 1 +459 val_459 2008-04-08 1 +459 val_459 2008-04-08 1 +459 val_459 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 1 +466 val_466 2008-04-08 1 +466 val_466 2008-04-08 1 +466 val_466 2008-04-08 1 +477 val_477 2008-04-08 0 +477 val_477 2008-04-08 0 +477 val_477 2008-04-08 0 +477 val_477 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +484 val_484 2008-04-08 0 +484 val_484 2008-04-08 0 +484 val_484 2008-04-08 0 +484 val_484 2008-04-08 0 491 val_491 2008-04-08 1 491 val_491 2008-04-08 1 491 val_491 2008-04-08 1 491 val_491 2008-04-08 1 +495 val_495 2008-04-08 0 +495 val_495 2008-04-08 0 +495 val_495 2008-04-08 0 +495 val_495 2008-04-08 0 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +51 val_51 2008-04-08 1 +66 val_66 2008-04-08 1 +66 val_66 2008-04-08 1 +66 val_66 2008-04-08 1 +66 val_66 2008-04-08 1 +77 val_77 2008-04-08 0 +77 val_77 2008-04-08 0 +77 val_77 2008-04-08 0 +77 val_77 2008-04-08 0 +8 val_8 2008-04-08 0 +8 val_8 2008-04-08 0 +8 val_8 2008-04-08 0 +8 val_8 2008-04-08 0 +80 val_80 2008-04-08 0 +80 val_80 2008-04-08 0 +80 val_80 2008-04-08 0 +80 val_80 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 1 +95 val_95 2008-04-08 1 +95 val_95 2008-04-08 1 +95 val_95 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -2604,22 +2645,21 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string), _col3 (type: int) + key expressions: _col3 (type: int), _bucket_number (type: string), _col0 (type: int), _col1 (type: string) + sort order: ++++ + Map-reduce partition columns: _col3 (type: int) + value expressions: _col2 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE + expressions: KEY._col0 (type: int), KEY._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: int), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number File Output Operator compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -2674,123 +2714,123 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8@hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@hr=1 #### A masked pattern was here #### 0 val_0 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -33 val_33 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -103 val_103 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -129 val_129 2008-04-08 0 -158 val_158 2008-04-08 0 -187 val_187 2008-04-08 0 -190 val_190 2008-04-08 0 -338 val_338 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -437 val_437 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -477 val_477 2008-04-08 0 -4 val_4 2008-04-08 0 -80 val_80 2008-04-08 0 -95 val_95 2008-04-08 0 -143 val_143 2008-04-08 0 -275 val_275 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -367 val_367 2008-04-08 0 -77 val_77 2008-04-08 0 -125 val_125 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -389 val_389 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -484 val_484 2008-04-08 0 -495 val_495 2008-04-08 0 -8 val_8 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -305 val_305 2008-04-08 0 -309 val_309 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -419 val_419 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -282 val_282 2008-04-08 0 -400 val_400 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 0 val_0 2008-04-08 1 0 val_0 2008-04-08 1 -44 val_44 2008-04-08 1 +103 val_103 2008-04-08 0 103 val_103 2008-04-08 1 -129 val_129 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 -466 val_466 2008-04-08 1 -95 val_95 2008-04-08 1 -286 val_286 2008-04-08 1 -367 val_367 2008-04-08 1 -491 val_491 2008-04-08 1 +11 val_11 2008-04-08 1 114 val_114 2008-04-08 1 +118 val_118 2008-04-08 0 +118 val_118 2008-04-08 0 +125 val_125 2008-04-08 0 125 val_125 2008-04-08 1 +129 val_129 2008-04-08 0 +129 val_129 2008-04-08 1 136 val_136 2008-04-08 1 +143 val_143 2008-04-08 0 +15 val_15 2008-04-08 1 +15 val_15 2008-04-08 1 +150 val_150 2008-04-08 1 +158 val_158 2008-04-08 0 165 val_165 2008-04-08 1 165 val_165 2008-04-08 1 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 0 +169 val_169 2008-04-08 1 +169 val_169 2008-04-08 1 +172 val_172 2008-04-08 0 +172 val_172 2008-04-08 0 176 val_176 2008-04-08 1 176 val_176 2008-04-08 1 +183 val_183 2008-04-08 1 +187 val_187 2008-04-08 0 +187 val_187 2008-04-08 1 +187 val_187 2008-04-08 1 +19 val_19 2008-04-08 1 +190 val_190 2008-04-08 0 +194 val_194 2008-04-08 1 202 val_202 2008-04-08 1 +213 val_213 2008-04-08 0 +213 val_213 2008-04-08 0 217 val_217 2008-04-08 1 217 val_217 2008-04-08 1 -323 val_323 2008-04-08 1 -455 val_455 2008-04-08 1 -459 val_459 2008-04-08 1 -11 val_11 2008-04-08 1 -66 val_66 2008-04-08 1 -150 val_150 2008-04-08 1 -183 val_183 2008-04-08 1 +224 val_224 2008-04-08 0 +224 val_224 2008-04-08 0 +228 val_228 2008-04-08 1 235 val_235 2008-04-08 1 239 val_239 2008-04-08 1 239 val_239 2008-04-08 1 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +257 val_257 2008-04-08 1 +26 val_26 2008-04-08 0 +26 val_26 2008-04-08 0 +260 val_260 2008-04-08 1 +275 val_275 2008-04-08 0 +282 val_282 2008-04-08 0 +282 val_282 2008-04-08 1 +286 val_286 2008-04-08 1 +305 val_305 2008-04-08 0 +309 val_309 2008-04-08 0 309 val_309 2008-04-08 1 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +316 val_316 2008-04-08 0 +323 val_323 2008-04-08 1 +327 val_327 2008-04-08 0 +327 val_327 2008-04-08 0 327 val_327 2008-04-08 1 +33 val_33 2008-04-08 0 +338 val_338 2008-04-08 0 +341 val_341 2008-04-08 0 345 val_345 2008-04-08 1 +356 val_356 2008-04-08 1 +367 val_367 2008-04-08 0 +367 val_367 2008-04-08 1 +37 val_37 2008-04-08 1 +37 val_37 2008-04-08 1 +374 val_374 2008-04-08 1 +378 val_378 2008-04-08 1 +389 val_389 2008-04-08 0 392 val_392 2008-04-08 1 +396 val_396 2008-04-08 0 396 val_396 2008-04-08 1 396 val_396 2008-04-08 1 +4 val_4 2008-04-08 0 +400 val_400 2008-04-08 0 +404 val_404 2008-04-08 0 +404 val_404 2008-04-08 0 +411 val_411 2008-04-08 1 +419 val_419 2008-04-08 0 +437 val_437 2008-04-08 0 +44 val_44 2008-04-08 1 +444 val_444 2008-04-08 1 +448 val_448 2008-04-08 0 +455 val_455 2008-04-08 1 +459 val_459 2008-04-08 0 +459 val_459 2008-04-08 1 462 val_462 2008-04-08 1 462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -19 val_19 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 0 +466 val_466 2008-04-08 1 +477 val_477 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +480 val_480 2008-04-08 0 +484 val_484 2008-04-08 0 +491 val_491 2008-04-08 1 +495 val_495 2008-04-08 0 51 val_51 2008-04-08 1 51 val_51 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -228 val_228 2008-04-08 1 -257 val_257 2008-04-08 1 -282 val_282 2008-04-08 1 -356 val_356 2008-04-08 1 -374 val_374 2008-04-08 1 -378 val_378 2008-04-08 1 -411 val_411 2008-04-08 1 -444 val_444 2008-04-08 1 +66 val_66 2008-04-08 1 +77 val_77 2008-04-08 0 +8 val_8 2008-04-08 0 +80 val_80 2008-04-08 0 +84 val_84 2008-04-08 0 +84 val_84 2008-04-08 0 +95 val_95 2008-04-08 0 +95 val_95 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -2827,6 +2867,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2838,16 +2881,28 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.srcbucket_mapjoin_n8 + Reduce Output Operator + key expressions: _col2 (type: string) + sort order: + + Map-reduce partition columns: _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -2892,124 +2947,124 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 #### A masked pattern was here #### -165 val_165 2008-04-08 -484 val_484 2008-04-08 -150 val_150 2008-04-08 -224 val_224 2008-04-08 -66 val_66 2008-04-08 -213 val_213 2008-04-08 -374 val_374 2008-04-08 -495 val_495 2008-04-08 -37 val_37 2008-04-08 -327 val_327 2008-04-08 -15 val_15 2008-04-08 -338 val_338 2008-04-08 -459 val_459 2008-04-08 -466 val_466 2008-04-08 -396 val_396 2008-04-08 -309 val_309 2008-04-08 -367 val_367 2008-04-08 0 val_0 2008-04-08 -455 val_455 2008-04-08 -316 val_316 2008-04-08 -345 val_345 2008-04-08 +0 val_0 2008-04-08 +0 val_0 2008-04-08 +103 val_103 2008-04-08 +103 val_103 2008-04-08 +11 val_11 2008-04-08 +114 val_114 2008-04-08 +118 val_118 2008-04-08 +118 val_118 2008-04-08 +125 val_125 2008-04-08 +125 val_125 2008-04-08 129 val_129 2008-04-08 -378 val_378 2008-04-08 -4 val_4 2008-04-08 -356 val_356 2008-04-08 +129 val_129 2008-04-08 +136 val_136 2008-04-08 +143 val_143 2008-04-08 +15 val_15 2008-04-08 +15 val_15 2008-04-08 +150 val_150 2008-04-08 +158 val_158 2008-04-08 +165 val_165 2008-04-08 +165 val_165 2008-04-08 169 val_169 2008-04-08 -125 val_125 2008-04-08 -437 val_437 2008-04-08 -286 val_286 2008-04-08 -187 val_187 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +169 val_169 2008-04-08 +172 val_172 2008-04-08 +172 val_172 2008-04-08 176 val_176 2008-04-08 -459 val_459 2008-04-08 -51 val_51 2008-04-08 -103 val_103 2008-04-08 -239 val_239 2008-04-08 -213 val_213 2008-04-08 176 val_176 2008-04-08 -275 val_275 2008-04-08 -260 val_260 2008-04-08 -404 val_404 2008-04-08 +183 val_183 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +187 val_187 2008-04-08 +19 val_19 2008-04-08 +190 val_190 2008-04-08 +194 val_194 2008-04-08 +202 val_202 2008-04-08 +213 val_213 2008-04-08 +213 val_213 2008-04-08 217 val_217 2008-04-08 -84 val_84 2008-04-08 -466 val_466 2008-04-08 -8 val_8 2008-04-08 -411 val_411 2008-04-08 -172 val_172 2008-04-08 -129 val_129 2008-04-08 -158 val_158 2008-04-08 -0 val_0 2008-04-08 +217 val_217 2008-04-08 +224 val_224 2008-04-08 +224 val_224 2008-04-08 +228 val_228 2008-04-08 +235 val_235 2008-04-08 +239 val_239 2008-04-08 +239 val_239 2008-04-08 +242 val_242 2008-04-08 +242 val_242 2008-04-08 +257 val_257 2008-04-08 26 val_26 2008-04-08 -165 val_165 2008-04-08 -327 val_327 2008-04-08 -51 val_51 2008-04-08 -404 val_404 2008-04-08 -95 val_95 2008-04-08 +26 val_26 2008-04-08 +260 val_260 2008-04-08 +275 val_275 2008-04-08 282 val_282 2008-04-08 -187 val_187 2008-04-08 -316 val_316 2008-04-08 -169 val_169 2008-04-08 -77 val_77 2008-04-08 -0 val_0 2008-04-08 -118 val_118 2008-04-08 282 val_282 2008-04-08 -419 val_419 2008-04-08 -15 val_15 2008-04-08 -118 val_118 2008-04-08 -19 val_19 2008-04-08 -224 val_224 2008-04-08 +286 val_286 2008-04-08 +305 val_305 2008-04-08 309 val_309 2008-04-08 -389 val_389 2008-04-08 +309 val_309 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +316 val_316 2008-04-08 +323 val_323 2008-04-08 +327 val_327 2008-04-08 +327 val_327 2008-04-08 327 val_327 2008-04-08 -242 val_242 2008-04-08 -392 val_392 2008-04-08 -242 val_242 2008-04-08 -396 val_396 2008-04-08 -95 val_95 2008-04-08 -11 val_11 2008-04-08 -143 val_143 2008-04-08 -228 val_228 2008-04-08 33 val_33 2008-04-08 -103 val_103 2008-04-08 +338 val_338 2008-04-08 +341 val_341 2008-04-08 +345 val_345 2008-04-08 +356 val_356 2008-04-08 367 val_367 2008-04-08 -239 val_239 2008-04-08 -480 val_480 2008-04-08 -202 val_202 2008-04-08 -316 val_316 2008-04-08 -235 val_235 2008-04-08 -80 val_80 2008-04-08 -44 val_44 2008-04-08 -466 val_466 2008-04-08 -257 val_257 2008-04-08 -190 val_190 2008-04-08 -114 val_114 2008-04-08 +367 val_367 2008-04-08 +37 val_37 2008-04-08 +37 val_37 2008-04-08 +374 val_374 2008-04-08 +378 val_378 2008-04-08 +389 val_389 2008-04-08 +392 val_392 2008-04-08 396 val_396 2008-04-08 -217 val_217 2008-04-08 -125 val_125 2008-04-08 -187 val_187 2008-04-08 -480 val_480 2008-04-08 -491 val_491 2008-04-08 -305 val_305 2008-04-08 +396 val_396 2008-04-08 +396 val_396 2008-04-08 +4 val_4 2008-04-08 +400 val_400 2008-04-08 +404 val_404 2008-04-08 +404 val_404 2008-04-08 +411 val_411 2008-04-08 +419 val_419 2008-04-08 +437 val_437 2008-04-08 +44 val_44 2008-04-08 444 val_444 2008-04-08 -169 val_169 2008-04-08 -323 val_323 2008-04-08 -480 val_480 2008-04-08 -136 val_136 2008-04-08 -172 val_172 2008-04-08 +448 val_448 2008-04-08 +455 val_455 2008-04-08 +459 val_459 2008-04-08 +459 val_459 2008-04-08 462 val_462 2008-04-08 -26 val_26 2008-04-08 462 val_462 2008-04-08 -341 val_341 2008-04-08 -183 val_183 2008-04-08 -84 val_84 2008-04-08 -37 val_37 2008-04-08 -448 val_448 2008-04-08 -194 val_194 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 +466 val_466 2008-04-08 477 val_477 2008-04-08 -169 val_169 2008-04-08 -400 val_400 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +480 val_480 2008-04-08 +484 val_484 2008-04-08 +491 val_491 2008-04-08 +495 val_495 2008-04-08 +51 val_51 2008-04-08 +51 val_51 2008-04-08 +66 val_66 2008-04-08 +77 val_77 2008-04-08 +8 val_8 2008-04-08 +80 val_80 2008-04-08 +84 val_84 2008-04-08 +84 val_84 2008-04-08 +95 val_95 2008-04-08 +95 val_95 2008-04-08 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -3042,6 +3097,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -3053,17 +3111,29 @@ STAGE PLANS: expressions: id (type: int), name (type: string), dept (type: string), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 24 Data size: 9024 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 24 Data size: 9024 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_test_txn - Write Type: INSERT + Reduce Output Operator + key expressions: _col3 (type: int) + sort order: + + Map-reduce partition columns: _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 24 Data size: 9024 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_test_txn + Write Type: INSERT Stage: Stage-2 Dependency Collection @@ -3118,6 +3188,6 @@ POSTHOOK: Input: default@orc_test_txn@year=2016 POSTHOOK: Input: default@orc_test_txn@year=2017 POSTHOOK: Input: default@orc_test_txn@year=2018 #### A masked pattern was here #### -9 Harris CSE 2017 -8 Henry CSE 2016 10 Haley CSE 2018 +8 Henry CSE 2016 +9 Harris CSE 2017 diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index c024062c74..9255be7dce 100644 --- a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -52,52 +52,28 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll') - keys: value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct) - Execution mode: llap + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: string) + Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part5 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out index 2c838b6a15..4ed30ee8a0 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out @@ -60,7 +60,6 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) Execution mode: vectorized, llap LLAP IO: may be used (ACID table) @@ -70,7 +69,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: string), KEY._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED diff --git a/ql/src/test/results/clientpositive/llap/mm_dp.q.out b/ql/src/test/results/clientpositive/llap/mm_dp.q.out index b23a3d720b..6ad05e9ed1 100644 --- a/ql/src/test/results/clientpositive/llap/mm_dp.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_dp.q.out @@ -149,7 +149,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _bucket_number (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 2605 Data size: 268315 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs @@ -159,11 +158,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY._col2 (type: int), KEY._col3 (type: string), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number - Statistics: Num rows: 2605 Data size: 747635 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 2605 Data size: 747635 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2605 Data size: 268315 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out index 25426bf5bf..1389de0c83 100644 --- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out @@ -772,7 +772,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5339 + totalSize 5336 #### A masked pattern was here #### # Storage Information @@ -866,7 +866,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5339 + totalSize 5336 #### A masked pattern was here #### # Storage Information @@ -1009,7 +1009,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5339 + totalSize 5336 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out index d4f95176ab..5bb6432d72 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -76,52 +76,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n1 Stage: Stage-2 Dependency Collection @@ -164,13 +140,8 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 6 items --rw-r--r-- 3 ### USER ### ### GROUP ### 555 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 562 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 561 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 496 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 554 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 478 ### HDFS DATE ### hdfs://### HDFS PATH ### +Found 1 items +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1359 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -212,52 +183,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n1 Stage: Stage-8 Conditional Operator @@ -346,7 +293,7 @@ POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1360 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1359 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -388,52 +335,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n1 Stage: Stage-8 Conditional Operator @@ -514,7 +437,7 @@ POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2461 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1359 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1_n1 WHERE ds='1' diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 95b352a1b3..d943b40374 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -76,52 +76,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 Stage: Stage-2 Dependency Collection @@ -164,10 +140,8 @@ POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)sr POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 3 items --rw-r--r-- 3 ### USER ### ### GROUP ### 947 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 875 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 856 ### HDFS DATE ### hdfs://### HDFS PATH ### +Found 1 items +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1754 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -209,52 +183,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b Stage: Stage-8 Conditional Operator @@ -385,52 +335,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c Stage: Stage-8 Conditional Operator @@ -511,7 +437,7 @@ POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2409 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1754 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1 WHERE ds='1' @@ -618,7 +544,7 @@ POSTHOOK: type: ALTER_PARTITION_MERGE POSTHOOK: Input: default@orcfile_merge1 POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0 Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2409 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1754 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1c WHERE ds='1' @@ -698,80 +624,41 @@ Type: struct Stripe Statistics: Stripe 1: - Column 0: count: 90 hasNull: false - Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736 - Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612 - Stripe 2: - Column 0: count: 78 hasNull: false - Column 1: count: 78 hasNull: false bytesOnDisk: 161 min: 0 max: 497 sum: 18371 - Column 2: count: 78 hasNull: false bytesOnDisk: 380 min: val_0 max: val_95 sum: 529 - Stripe 3: - Column 0: count: 74 hasNull: false - Column 1: count: 74 hasNull: false bytesOnDisk: 153 min: 2 max: 493 sum: 19663 - Column 2: count: 74 hasNull: false bytesOnDisk: 363 min: val_105 max: val_97 sum: 505 + Column 0: count: 242 hasNull: false + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 File Statistics: Column 0: count: 242 hasNull: false - Column 1: count: 242 hasNull: false bytesOnDisk: 499 min: 0 max: 497 sum: 60770 - Column 2: count: 242 hasNull: false bytesOnDisk: 1171 min: val_0 max: val_97 sum: 1646 + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 Stripes: - Stripe: offset: 3 data: 613 rows: 90 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 3 length 11 - Stream: column 1 section ROW_INDEX start: 14 length 27 - Stream: column 2 section ROW_INDEX start: 41 length 38 - Stream: column 1 section DATA start: 79 length 185 - Stream: column 2 section DATA start: 264 length 377 - Stream: column 2 section LENGTH start: 641 length 51 + Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77 + Stream: column 0 section ROW_INDEX start: 3 length 12 + Stream: column 1 section ROW_INDEX start: 15 length 28 + Stream: column 2 section ROW_INDEX start: 43 length 37 + Stream: column 1 section DATA start: 80 length 489 + Stream: column 2 section DATA start: 569 length 247 + Stream: column 2 section LENGTH start: 816 length 71 + Stream: column 2 section DICTIONARY_DATA start: 887 length 592 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 + Encoding column 2: DICTIONARY_V2[153] Row group indices for column 0: - Entry 0: count: 90 hasNull: false positions: + Entry 0: count: 242 hasNull: false positions: Row group indices for column 1: - Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0 + Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0 Row group indices for column 2: - Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0 - Stripe: offset: 753 data: 541 rows: 78 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 753 length 11 - Stream: column 1 section ROW_INDEX start: 764 length 27 - Stream: column 2 section ROW_INDEX start: 791 length 38 - Stream: column 1 section DATA start: 829 length 161 - Stream: column 2 section DATA start: 990 length 332 - Stream: column 2 section LENGTH start: 1322 length 48 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 78 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 78 hasNull: false min: val_0 max: val_95 sum: 529 positions: 0,0,0,0,0 - Stripe: offset: 1431 data: 516 rows: 74 tail: 61 index: 78 - Stream: column 0 section ROW_INDEX start: 1431 length 11 - Stream: column 1 section ROW_INDEX start: 1442 length 27 - Stream: column 2 section ROW_INDEX start: 1469 length 40 - Stream: column 1 section DATA start: 1509 length 153 - Stream: column 2 section DATA start: 1662 length 331 - Stream: column 2 section LENGTH start: 1993 length 32 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 74 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 74 hasNull: false min: val_105 max: val_97 sum: 505 positions: 0,0,0,0,0 + Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 2409 bytes +File length: 1754 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ -- END ORC FILE DUMP -- -86 val_86 1 0 +2 val_2 1 0 PREHOOK: query: select * from orcfile_merge1c where ds='1' and part='0' limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@orcfile_merge1c @@ -787,80 +674,41 @@ Type: struct Stripe Statistics: Stripe 1: - Column 0: count: 90 hasNull: false - Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736 - Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612 - Stripe 2: - Column 0: count: 78 hasNull: false - Column 1: count: 78 hasNull: false bytesOnDisk: 161 min: 0 max: 497 sum: 18371 - Column 2: count: 78 hasNull: false bytesOnDisk: 380 min: val_0 max: val_95 sum: 529 - Stripe 3: - Column 0: count: 74 hasNull: false - Column 1: count: 74 hasNull: false bytesOnDisk: 153 min: 2 max: 493 sum: 19663 - Column 2: count: 74 hasNull: false bytesOnDisk: 363 min: val_105 max: val_97 sum: 505 + Column 0: count: 242 hasNull: false + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 File Statistics: Column 0: count: 242 hasNull: false - Column 1: count: 242 hasNull: false bytesOnDisk: 499 min: 0 max: 497 sum: 60770 - Column 2: count: 242 hasNull: false bytesOnDisk: 1171 min: val_0 max: val_97 sum: 1646 + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 Stripes: - Stripe: offset: 3 data: 613 rows: 90 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 3 length 11 - Stream: column 1 section ROW_INDEX start: 14 length 27 - Stream: column 2 section ROW_INDEX start: 41 length 38 - Stream: column 1 section DATA start: 79 length 185 - Stream: column 2 section DATA start: 264 length 377 - Stream: column 2 section LENGTH start: 641 length 51 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 90 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0 - Stripe: offset: 753 data: 541 rows: 78 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 753 length 11 - Stream: column 1 section ROW_INDEX start: 764 length 27 - Stream: column 2 section ROW_INDEX start: 791 length 38 - Stream: column 1 section DATA start: 829 length 161 - Stream: column 2 section DATA start: 990 length 332 - Stream: column 2 section LENGTH start: 1322 length 48 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 78 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 78 hasNull: false min: val_0 max: val_95 sum: 529 positions: 0,0,0,0,0 - Stripe: offset: 1431 data: 516 rows: 74 tail: 61 index: 78 - Stream: column 0 section ROW_INDEX start: 1431 length 11 - Stream: column 1 section ROW_INDEX start: 1442 length 27 - Stream: column 2 section ROW_INDEX start: 1469 length 40 - Stream: column 1 section DATA start: 1509 length 153 - Stream: column 2 section DATA start: 1662 length 331 - Stream: column 2 section LENGTH start: 1993 length 32 + Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77 + Stream: column 0 section ROW_INDEX start: 3 length 12 + Stream: column 1 section ROW_INDEX start: 15 length 28 + Stream: column 2 section ROW_INDEX start: 43 length 37 + Stream: column 1 section DATA start: 80 length 489 + Stream: column 2 section DATA start: 569 length 247 + Stream: column 2 section LENGTH start: 816 length 71 + Stream: column 2 section DICTIONARY_DATA start: 887 length 592 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 + Encoding column 2: DICTIONARY_V2[153] Row group indices for column 0: - Entry 0: count: 74 hasNull: false positions: + Entry 0: count: 242 hasNull: false positions: Row group indices for column 1: - Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0 + Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0 Row group indices for column 2: - Entry 0: count: 74 hasNull: false min: val_105 max: val_97 sum: 505 positions: 0,0,0,0,0 + Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 2409 bytes +File length: 1754 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ -- END ORC FILE DUMP -- -86 val_86 1 0 +2 val_2 1 0 PREHOOK: query: DROP TABLE orcfile_merge1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@orcfile_merge1 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out index 430ce16239..7265292d09 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -50,52 +50,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge2a_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) - outputColumnNames: key, value, one, two, three - Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: one (type: string), two (type: string), three (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: struct), _col4 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a_n0 Stage: Stage-2 Dependency Collection @@ -196,7 +172,7 @@ POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=1).value SIMPL POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-r--r-- 3 ### USER ### ### GROUP ### 349 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 349 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a_n0 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out index 1f67f7b828..634f1399bf 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -52,52 +52,28 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a_n0 - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double) - outputColumnNames: userid, string1, subtype, decimal1, ts, st - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll') - keys: st (type: double) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) Execution mode: llap LLAP IO: all inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4) - keys: KEY._col0 (type: double) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a_n0 Stage: Stage-2 Dependency Collection @@ -272,52 +248,28 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a_n0 - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double) - outputColumnNames: userid, string1, subtype, decimal1, ts, st - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll') - keys: st (type: double) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) Execution mode: llap LLAP IO: all inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4) - keys: KEY._col0 (type: double) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a_n0 Stage: Stage-8 Conditional Operator diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out index 7cd5ba4210..a912216dc5 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out @@ -76,52 +76,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n0 Stage: Stage-2 Dependency Collection @@ -164,7 +140,7 @@ POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 6 items +Found 1 items #### A masked pattern was here #### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n0 PARTITION (ds='1', part) @@ -207,52 +183,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n0 Stage: Stage-8 Conditional Operator @@ -383,52 +335,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n0 Stage: Stage-8 Conditional Operator diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index 5c5bc8ea95..ee13fafc61 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -52,52 +52,28 @@ STAGE PLANS: expressions: userid (type: bigint), string1 (type: string), subtype (type: double), decimal1 (type: decimal(38,0)), ts (type: timestamp), subtype (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orc_merge5a - Select Operator - expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp), _col5 (type: double) - outputColumnNames: userid, string1, subtype, decimal1, ts, st - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(userid, 'hll'), compute_stats(string1, 'hll'), compute_stats(subtype, 'hll'), compute_stats(decimal1, 'hll'), compute_stats(ts, 'hll') - keys: st (type: double) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: double) - sort order: + - Map-reduce partition columns: _col0 (type: double) - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reduce Output Operator + key expressions: _col5 (type: double) + sort order: + + Map-reduce partition columns: _col5 (type: double) + value expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: double), _col3 (type: decimal(38,0)), _col4 (type: timestamp) Execution mode: llap LLAP IO: all inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3), compute_stats(VALUE._col4) - keys: KEY._col0 (type: double) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: bigint), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: decimal(38,0)), VALUE._col4 (type: timestamp), KEY._col5 (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: double) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 352 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orc_merge5a Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 083ad3074a..dfe4422583 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -2594,24 +2594,16 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Select Operator - expressions: _col1 (type: double) - outputColumnNames: _col1 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: floor(_col1) (type: bigint) + Group By Operator + keys: _col0 (type: bigint) + mode: hash outputColumnNames: _col0 - Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - keys: _col0 (type: bigint) - mode: hash - outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: bigint) - sort order: + - Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Reducer 6 Execution mode: vectorized, llap Reduce Operator Tree: diff --git a/ql/src/test/results/clientpositive/llap/tez_dml.q.out b/ql/src/test/results/clientpositive/llap/tez_dml.q.out index 67d4bf32e4..a9b3172777 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dml.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dml.q.out @@ -466,52 +466,28 @@ STAGE PLANS: expressions: value (type: string), cnt (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.tmp_src_part - Select Operator - expressions: _col0 (type: string), UDFToInteger(_col1) (type: int) - outputColumnNames: c, d - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(c, 'hll') - keys: d (type: int) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct) - Execution mode: llap + Reduce Output Operator + key expressions: _col1 (type: bigint) + sort order: + + Map-reduce partition columns: _col1 (type: bigint) + value expressions: _col0 (type: string) + Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: int) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 154 Data size: 28228 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col0 (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 154 Data size: 28228 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 154 Data size: 28228 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 309 Data size: 56640 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmp_src_part Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out b/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out index ca8232e33d..28977d78f4 100644 --- a/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_count_distinct.q.out @@ -1316,7 +1316,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: web_sales - Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true Select Operator @@ -1326,7 +1326,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [16] - Statistics: Num rows: 2000 Data size: 8000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE Group By Operator Group By Vectorization: className: VectorGroupByOperator @@ -1338,7 +1338,7 @@ STAGE PLANS: keys: ws_order_number (type: int) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 170 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -1347,7 +1347,7 @@ STAGE PLANS: className: VectorReduceSinkLongOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 170 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 3511604 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: @@ -1379,7 +1379,7 @@ STAGE PLANS: keys: KEY._col0 (type: int) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 170 Data size: 680 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 1755802 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col0) Group By Vectorization: @@ -1391,14 +1391,14 @@ STAGE PLANS: projectedOutputColumnNums: [0] mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: Reduce Sink Vectorization: className: VectorReduceSinkEmptyKeyOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 3 Execution mode: vectorized, llap @@ -1420,13 +1420,13 @@ STAGE PLANS: projectedOutputColumnNums: [0] mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out index 4711f35165..2ab11daa46 100644 --- a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out @@ -57,139 +57,139 @@ POSTHOOK: Input: default@flights_tiny_orc Baltimore New York 2010-10-20 2010-10-20 07:00:00 -30.0 1064 Baltimore New York 2010-10-20 2010-10-20 07:00:00 23.0 1142 Baltimore New York 2010-10-20 2010-10-20 07:00:00 6.0 1599 -Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 -Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 -Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 -Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 Baltimore New York 2010-10-21 2010-10-21 07:00:00 105.0 1142 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 Baltimore New York 2010-10-21 2010-10-21 07:00:00 28.0 1599 -Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 -Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 -Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 Baltimore New York 2010-10-22 2010-10-22 07:00:00 -12.0 1064 -Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 Baltimore New York 2010-10-22 2010-10-22 07:00:00 18.0 1599 -Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 -Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 -Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 Baltimore New York 2010-10-23 2010-10-23 07:00:00 -10.0 1805 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 Baltimore New York 2010-10-23 2010-10-23 07:00:00 6.0 3171 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 -Chicago New York 2010-10-23 2010-10-23 07:00:00 11.0 3085 Baltimore New York 2010-10-24 2010-10-24 07:00:00 12.0 1599 Baltimore New York 2010-10-24 2010-10-24 07:00:00 20.0 2571 -Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -17.0 1610 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -3.0 3198 Baltimore New York 2010-10-25 2010-10-25 07:00:00 -25.0 1064 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 Baltimore New York 2010-10-25 2010-10-25 07:00:00 106.0 1599 -Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 43.0 1531 -Chicago New York 2010-10-25 2010-10-25 07:00:00 6.0 1610 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 Baltimore New York 2010-10-26 2010-10-26 07:00:00 -22.0 1064 Baltimore New York 2010-10-26 2010-10-26 07:00:00 123.0 1142 Baltimore New York 2010-10-26 2010-10-26 07:00:00 90.0 1599 -Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 29.0 1531 -Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 -Chicago New York 2010-10-26 2010-10-26 07:00:00 6.0 3198 Baltimore New York 2010-10-27 2010-10-27 07:00:00 -18.0 1064 Baltimore New York 2010-10-27 2010-10-27 07:00:00 49.0 1142 Baltimore New York 2010-10-27 2010-10-27 07:00:00 92.0 1599 -Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 +Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 +Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 +Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 +Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 11.0 3085 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -17.0 1610 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -3.0 3198 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-25 2010-10-25 07:00:00 43.0 1531 +Chicago New York 2010-10-25 2010-10-25 07:00:00 6.0 1610 +Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 29.0 1531 +Chicago New York 2010-10-26 2010-10-26 07:00:00 6.0 3198 Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 Chicago New York 2010-10-27 2010-10-27 07:00:00 70.0 1531 Chicago New York 2010-10-27 2010-10-27 07:00:00 8.0 1610 -Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 897 -Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 Chicago New York 2010-10-28 2010-10-28 07:00:00 3.0 1610 -Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -11.0 897 -Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -18.0 1610 Chicago New York 2010-10-29 2010-10-29 07:00:00 -4.0 3198 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 3085 -Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 -Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 Chicago New York 2010-10-31 2010-10-31 07:00:00 -18.0 897 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 Chicago New York 2010-10-31 2010-10-31 07:00:00 -22.0 1610 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 -Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 -Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 -Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 -Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 -Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 +Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 Cleveland New York 2010-10-24 2010-10-24 07:00:00 -11.0 2630 Cleveland New York 2010-10-24 2010-10-24 07:00:00 -20.0 2646 Cleveland New York 2010-10-24 2010-10-24 07:00:00 -9.0 3014 -Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 -Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 -Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 -Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 -Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 -Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 -Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 -Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 Washington New York 2010-10-20 2010-10-20 07:00:00 -2.0 7291 Washington New York 2010-10-21 2010-10-21 07:00:00 22.0 7291 Washington New York 2010-10-23 2010-10-23 07:00:00 -16.0 7274 +Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 +Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 +Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 Washington New York 2010-10-24 2010-10-24 07:00:00 -26.0 7282 Washington New York 2010-10-25 2010-10-25 07:00:00 9.0 7291 Washington New York 2010-10-26 2010-10-26 07:00:00 4.0 7291 Washington New York 2010-10-27 2010-10-27 07:00:00 26.0 7291 Washington New York 2010-10-28 2010-10-28 07:00:00 45.0 7291 Washington New York 2010-10-29 2010-10-29 07:00:00 1.0 7291 +Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 +Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 Washington New York 2010-10-31 2010-10-31 07:00:00 -18.0 7282 PREHOOK: query: select * from flights_tiny_orc sort by fl_num, fl_date limit 25 PREHOOK: type: QUERY @@ -201,29 +201,29 @@ POSTHOOK: Input: default@flights_tiny_orc #### A masked pattern was here #### Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 PREHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny_orc @@ -232,16 +232,16 @@ POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny_orc #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: explain vectorization expression @@ -390,29 +390,29 @@ POSTHOOK: Input: default@flights_tiny_orc #### A masked pattern was here #### Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 PREHOOK: query: explain vectorization expression select fl_date, count(*) from flights_tiny_orc group by fl_date PREHOOK: type: QUERY @@ -537,16 +537,16 @@ POSTHOOK: query: select fl_date, count(*) from flights_tiny_orc group by fl_date POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny_orc #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( @@ -690,142 +690,142 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### -Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 -Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 -Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 -Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 -Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 -Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 -Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 -Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 -Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 -Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 -Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 -Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 -Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 -Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 -Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 -Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 -Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 -Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 -Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 -Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 -Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 -Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 +Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 -Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 -Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 -Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 -Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 -Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 -Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 -Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 -Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 -Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 -Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 -Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 +Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 +Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 PREHOOK: query: select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 PREHOOK: type: QUERY @@ -861,29 +861,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 PREHOOK: query: select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny_orc_partitioned_date @@ -916,16 +916,16 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: explain vectorization expression @@ -979,7 +979,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_date - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -989,13 +989,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1050,142 +1050,142 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### -Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 -Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 -Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 -Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 -Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 -Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 -Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 -Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 -Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 -Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 -Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 -Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 -Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 -Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 -Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 -Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 -Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 -Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 -Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 -Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 -Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 -Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 +Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 -Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 -Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 -Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 -Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 -Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 -Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 -Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 -Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 -Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 -Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 -Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 -Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 +Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 +Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 +Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 +Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 PREHOOK: query: explain vectorization expression select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25 @@ -1242,7 +1242,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_date - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -1252,7 +1252,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: date) sort order: ++ @@ -1260,7 +1260,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 94360 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Execution mode: vectorized, llap @@ -1290,13 +1290,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: date) sort order: ++ @@ -1304,7 +1304,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Reducer 3 @@ -1323,19 +1323,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1381,29 +1381,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 PREHOOK: query: explain vectorization expression select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date PREHOOK: type: QUERY @@ -1576,16 +1576,16 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-29 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( @@ -1729,142 +1729,142 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 -Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 -Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 -Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 -Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 -Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 -Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 -Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 -Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 -Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 -Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 -Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 -Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 -Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 -Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 -Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 -Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 -Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 -Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 -Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 -Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 +Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 -Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 -Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 -Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 -Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 -Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 -Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 -Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 -Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 -Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 -Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 -Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 +Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 +Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 PREHOOK: query: select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 PREHOOK: type: QUERY @@ -1900,29 +1900,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- #### A masked pattern was here #### Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 PREHOOK: query: select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny_orc_partitioned_timestamp @@ -1955,18 +1955,18 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### +2010-10-20 07:00:00 11 2010-10-21 07:00:00 12 -2010-10-24 07:00:00 12 -2010-10-31 07:00:00 8 2010-10-22 07:00:00 11 2010-10-23 07:00:00 12 -2010-10-30 07:00:00 11 -2010-10-20 07:00:00 11 +2010-10-24 07:00:00 12 2010-10-25 07:00:00 12 2010-10-26 07:00:00 13 2010-10-27 07:00:00 11 2010-10-28 07:00:00 12 2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 PREHOOK: query: explain vectorization expression select * from flights_tiny_orc_partitioned_timestamp PREHOOK: type: QUERY @@ -2018,7 +2018,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_timestamp - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 96472 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -2028,13 +2028,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2082,149 +2082,149 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-23 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-24 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-25 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 -#### A masked pattern was here #### -Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 -Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 -Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 -Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 -Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 -Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 -Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 -Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 -Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 -Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 -Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 -Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 -Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 -Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 -Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 -Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 -Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 -Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 -Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 -Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 -Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 -Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 -Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 -Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 -Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 +Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 +Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 +Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 +Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 -Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 -Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 -Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 -Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 -Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 -Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 -Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 -Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 -Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 -Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 -Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 -Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 +Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 +Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 +Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 +Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 PREHOOK: query: explain vectorization expression select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25 @@ -2281,7 +2281,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_orc_partitioned_timestamp - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 96472 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -2291,7 +2291,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 96472 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: timestamp) sort order: ++ @@ -2299,7 +2299,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 96472 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Execution mode: vectorized, llap @@ -2329,13 +2329,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: timestamp) sort order: ++ @@ -2343,7 +2343,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Reducer 3 @@ -2362,19 +2362,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -2420,29 +2420,29 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- #### A masked pattern was here #### Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 PREHOOK: query: explain vectorization expression select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time PREHOOK: type: QUERY @@ -2615,18 +2615,18 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### +2010-10-20 07:00:00 11 2010-10-21 07:00:00 12 -2010-10-24 07:00:00 12 -2010-10-31 07:00:00 8 2010-10-22 07:00:00 11 2010-10-23 07:00:00 12 -2010-10-30 07:00:00 11 -2010-10-20 07:00:00 11 +2010-10-24 07:00:00 12 2010-10-25 07:00:00 12 2010-10-26 07:00:00 13 2010-10-27 07:00:00 11 2010-10-28 07:00:00 12 2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 PREHOOK: query: CREATE TABLE flights_tiny_parquet STORED AS PARQUET AS SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles') as fl_time, arr_delay, fl_num FROM flights_tiny_n1 @@ -2658,139 +2658,139 @@ POSTHOOK: Input: default@flights_tiny_parquet Baltimore New York 2010-10-20 2010-10-20 07:00:00 -30.0 1064 Baltimore New York 2010-10-20 2010-10-20 07:00:00 23.0 1142 Baltimore New York 2010-10-20 2010-10-20 07:00:00 6.0 1599 -Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 -Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 -Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 -Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 Baltimore New York 2010-10-21 2010-10-21 07:00:00 105.0 1142 +Baltimore New York 2010-10-21 2010-10-21 07:00:00 17.0 1064 Baltimore New York 2010-10-21 2010-10-21 07:00:00 28.0 1599 -Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 -Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 -Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 Baltimore New York 2010-10-22 2010-10-22 07:00:00 -12.0 1064 -Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 Baltimore New York 2010-10-22 2010-10-22 07:00:00 18.0 1599 +Baltimore New York 2010-10-22 2010-10-22 07:00:00 54.0 1142 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 -10.0 1805 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 +Baltimore New York 2010-10-23 2010-10-23 07:00:00 6.0 3171 +Baltimore New York 2010-10-24 2010-10-24 07:00:00 12.0 1599 +Baltimore New York 2010-10-24 2010-10-24 07:00:00 20.0 2571 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 -25.0 1064 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 106.0 1599 +Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 -22.0 1064 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 123.0 1142 +Baltimore New York 2010-10-26 2010-10-26 07:00:00 90.0 1599 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 -18.0 1064 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 49.0 1142 +Baltimore New York 2010-10-27 2010-10-27 07:00:00 92.0 1599 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 +Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 +Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 +Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -2.0 3198 +Chicago New York 2010-10-20 2010-10-20 07:00:00 -6.0 1610 +Chicago New York 2010-10-20 2010-10-20 07:00:00 15.0 1531 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 +Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 -5.0 1610 +Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 51.0 3198 +Chicago New York 2010-10-21 2010-10-21 07:00:00 53.0 1531 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 +Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 +Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 16.0 1531 -Chicago New York 2010-10-22 2010-10-22 07:00:00 -6.0 1610 -Chicago New York 2010-10-22 2010-10-22 07:00:00 -11.0 3198 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 -10.0 1805 -Baltimore New York 2010-10-23 2010-10-23 07:00:00 6.0 3171 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 Chicago New York 2010-10-23 2010-10-23 07:00:00 11.0 3085 -Baltimore New York 2010-10-24 2010-10-24 07:00:00 12.0 1599 -Baltimore New York 2010-10-24 2010-10-24 07:00:00 20.0 2571 -Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-24 2010-10-24 07:00:00 -17.0 1610 Chicago New York 2010-10-24 2010-10-24 07:00:00 -3.0 3198 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 -25.0 1064 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 92.0 1142 -Baltimore New York 2010-10-25 2010-10-25 07:00:00 106.0 1599 -Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 -5.0 1531 +Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 +Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 Chicago New York 2010-10-25 2010-10-25 07:00:00 43.0 1531 Chicago New York 2010-10-25 2010-10-25 07:00:00 6.0 1610 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -16.0 3198 -Baltimore New York 2010-10-26 2010-10-26 07:00:00 -22.0 1064 -Baltimore New York 2010-10-26 2010-10-26 07:00:00 123.0 1142 -Baltimore New York 2010-10-26 2010-10-26 07:00:00 90.0 1599 -Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 +Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 Chicago New York 2010-10-26 2010-10-26 07:00:00 29.0 1531 -Chicago New York 2010-10-26 2010-10-26 07:00:00 -17.0 1610 Chicago New York 2010-10-26 2010-10-26 07:00:00 6.0 3198 -Baltimore New York 2010-10-27 2010-10-27 07:00:00 -18.0 1064 -Baltimore New York 2010-10-27 2010-10-27 07:00:00 49.0 1142 -Baltimore New York 2010-10-27 2010-10-27 07:00:00 92.0 1599 -Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 Chicago New York 2010-10-27 2010-10-27 07:00:00 70.0 1531 Chicago New York 2010-10-27 2010-10-27 07:00:00 8.0 1610 -Chicago New York 2010-10-27 2010-10-27 07:00:00 21.0 3198 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -4.0 1064 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1142 -Baltimore New York 2010-10-28 2010-10-28 07:00:00 -14.0 1599 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 +Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 897 -Chicago New York 2010-10-28 2010-10-28 07:00:00 -11.0 1531 Chicago New York 2010-10-28 2010-10-28 07:00:00 3.0 1610 -Chicago New York 2010-10-28 2010-10-28 07:00:00 -18.0 3198 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 -24.0 1064 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 21.0 1142 -Baltimore New York 2010-10-29 2010-10-29 07:00:00 -2.0 1599 -Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -11.0 897 -Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 +Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -18.0 1610 Chicago New York 2010-10-29 2010-10-29 07:00:00 -4.0 3198 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 -1.0 1805 -Baltimore New York 2010-10-30 2010-10-30 07:00:00 5.0 3171 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-29 2010-10-29 07:00:00 15.0 1531 Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 3085 -Baltimore New York 2010-10-31 2010-10-31 07:00:00 -1.0 1599 -Baltimore New York 2010-10-31 2010-10-31 07:00:00 -14.0 2571 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 Chicago New York 2010-10-31 2010-10-31 07:00:00 -18.0 897 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 Chicago New York 2010-10-31 2010-10-31 07:00:00 -22.0 1610 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -15.0 3198 -Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 -Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 -Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 -Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 -Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 -Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 -Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 -Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 -Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -4.0 1531 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 +Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 +Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 +Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 +Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 Cleveland New York 2010-10-24 2010-10-24 07:00:00 -11.0 2630 Cleveland New York 2010-10-24 2010-10-24 07:00:00 -20.0 2646 Cleveland New York 2010-10-24 2010-10-24 07:00:00 -9.0 3014 -Cleveland New York 2010-10-23 2010-10-23 07:00:00 -21.0 2932 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 1.0 2630 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 -25.0 2646 -Cleveland New York 2010-10-22 2010-10-22 07:00:00 -3.0 3014 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 3.0 2630 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 29.0 2646 -Cleveland New York 2010-10-21 2010-10-21 07:00:00 72.0 3014 -Cleveland New York 2010-10-20 2010-10-20 07:00:00 -8.0 2630 -Cleveland New York 2010-10-20 2010-10-20 07:00:00 -15.0 3014 -Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 -Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 -Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 -Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 -Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 +Cleveland New York 2010-10-24 2010-10-24 07:00:00 5.0 2254 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 -4.0 2630 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 42.0 3014 +Cleveland New York 2010-10-25 2010-10-25 07:00:00 81.0 2646 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -11.0 2662 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 -27.0 2646 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 13.0 3014 +Cleveland New York 2010-10-26 2010-10-26 07:00:00 4.0 2630 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 16.0 2630 +Cleveland New York 2010-10-27 2010-10-27 07:00:00 27.0 3014 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 -6.0 2646 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 1.0 3014 +Cleveland New York 2010-10-28 2010-10-28 07:00:00 3.0 2630 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -12.0 3014 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -19.0 2646 +Cleveland New York 2010-10-29 2010-10-29 07:00:00 -4.0 2630 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -12.0 2932 +Cleveland New York 2010-10-30 2010-10-30 07:00:00 -23.0 2018 Washington New York 2010-10-20 2010-10-20 07:00:00 -2.0 7291 Washington New York 2010-10-21 2010-10-21 07:00:00 22.0 7291 Washington New York 2010-10-23 2010-10-23 07:00:00 -16.0 7274 +Washington New York 2010-10-23 2010-10-23 07:00:00 -18.0 5917 +Washington New York 2010-10-23 2010-10-23 07:00:00 -21.0 5904 +Washington New York 2010-10-23 2010-10-23 07:00:00 -25.0 5832 Washington New York 2010-10-24 2010-10-24 07:00:00 -26.0 7282 Washington New York 2010-10-25 2010-10-25 07:00:00 9.0 7291 Washington New York 2010-10-26 2010-10-26 07:00:00 4.0 7291 Washington New York 2010-10-27 2010-10-27 07:00:00 26.0 7291 Washington New York 2010-10-28 2010-10-28 07:00:00 45.0 7291 Washington New York 2010-10-29 2010-10-29 07:00:00 1.0 7291 +Washington New York 2010-10-30 2010-10-30 07:00:00 -16.0 5917 +Washington New York 2010-10-30 2010-10-30 07:00:00 -27.0 5904 Washington New York 2010-10-31 2010-10-31 07:00:00 -18.0 7282 PREHOOK: query: select * from flights_tiny_parquet sort by fl_num, fl_date limit 25 PREHOOK: type: QUERY @@ -2802,29 +2802,29 @@ POSTHOOK: Input: default@flights_tiny_parquet #### A masked pattern was here #### Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 PREHOOK: query: select fl_date, count(*) from flights_tiny_parquet group by fl_date PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny_parquet @@ -2833,16 +2833,16 @@ POSTHOOK: query: select fl_date, count(*) from flights_tiny_parquet group by fl_ POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny_parquet #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: explain vectorization expression @@ -2991,29 +2991,29 @@ POSTHOOK: Input: default@flights_tiny_parquet #### A masked pattern was here #### Baltimore New York 2010-10-23 2010-10-23 07:00:00 18.0 272 Baltimore New York 2010-10-30 2010-10-30 07:00:00 14.0 272 +Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 Chicago New York 2010-10-20 2010-10-20 07:00:00 42.0 361 Chicago New York 2010-10-21 2010-10-21 07:00:00 142.0 361 +Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 Chicago New York 2010-10-22 2010-10-22 07:00:00 2.0 361 +Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 +Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 +Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 +Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-24 2010-10-24 07:00:00 10.0 361 +Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 +Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 Chicago New York 2010-10-25 2010-10-25 07:00:00 31.0 361 +Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 Chicago New York 2010-10-26 2010-10-26 07:00:00 12.0 361 +Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 Chicago New York 2010-10-27 2010-10-27 07:00:00 148.0 361 Chicago New York 2010-10-28 2010-10-28 07:00:00 2.0 361 Chicago New York 2010-10-29 2010-10-29 07:00:00 -12.0 361 -Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 -Chicago New York 2010-10-23 2010-10-23 07:00:00 3.0 384 -Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 -Chicago New York 2010-10-23 2010-10-23 07:00:00 32.0 426 Chicago New York 2010-10-30 2010-10-30 07:00:00 -10.0 426 -Chicago New York 2010-10-23 2010-10-23 07:00:00 1.0 650 Chicago New York 2010-10-30 2010-10-30 07:00:00 -5.0 650 -Chicago New York 2010-10-20 2010-10-20 07:00:00 24.0 897 -Chicago New York 2010-10-21 2010-10-21 07:00:00 77.0 897 -Chicago New York 2010-10-22 2010-10-22 07:00:00 24.0 897 -Chicago New York 2010-10-24 2010-10-24 07:00:00 113.0 897 -Chicago New York 2010-10-25 2010-10-25 07:00:00 -1.0 897 -Chicago New York 2010-10-26 2010-10-26 07:00:00 0.0 897 -Chicago New York 2010-10-27 2010-10-27 07:00:00 -11.0 897 +Chicago New York 2010-10-30 2010-10-30 07:00:00 -6.0 384 +Chicago New York 2010-10-31 2010-10-31 07:00:00 -25.0 361 PREHOOK: query: explain vectorization expression select fl_date, count(*) from flights_tiny_parquet group by fl_date PREHOOK: type: QUERY @@ -3138,16 +3138,16 @@ POSTHOOK: query: select fl_date, count(*) from flights_tiny_parquet group by fl_ POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny_parquet #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: CREATE TABLE flights_tiny_parquet_partitioned_date ( @@ -3291,142 +3291,142 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-2 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### -Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 -Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 -Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 +Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 +Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 +Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 +Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 +Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 -Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 -Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 -Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 -Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 -Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 -Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 -Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 -Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 -Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 -Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 -Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 -Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 -Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 -Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 -Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 -Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 -Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 -Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 +Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 +Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 -Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 -Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 -Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 -Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 -Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 -Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 -Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 -Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 +Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 -Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 -Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 -Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 -Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 -Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 PREHOOK: query: select * from flights_tiny_parquet_partitioned_date sort by fl_num, fl_date limit 25 PREHOOK: type: QUERY @@ -3462,29 +3462,29 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-3 #### A masked pattern was here #### Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 PREHOOK: query: select fl_date, count(*) from flights_tiny_parquet_partitioned_date group by fl_date PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny_parquet_partitioned_date @@ -3517,16 +3517,16 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-2 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: explain vectorization expression @@ -3580,7 +3580,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_parquet_partitioned_date - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63269 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -3590,13 +3590,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3651,142 +3651,142 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-2 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### -Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 -Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 -Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 -Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 -Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 -Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 -Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 -Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 -Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 -Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 -Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 -Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 -Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 -Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 -Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 -Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 -Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 -Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 -Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 -Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 -Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 -Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 +Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 +Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 +Baltimore New York 2010-10-21 07:00:00 105.0 1142 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 17.0 1064 2010-10-21 +Baltimore New York 2010-10-21 07:00:00 28.0 1599 2010-10-21 +Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 +Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 Baltimore New York 2010-10-23 07:00:00 -10.0 1805 2010-10-23 +Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-23 07:00:00 6.0 3171 2010-10-23 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 -Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 -Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 -Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 -Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 -Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 -Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 Baltimore New York 2010-10-24 07:00:00 12.0 1599 2010-10-24 Baltimore New York 2010-10-24 07:00:00 20.0 2571 2010-10-24 -Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 -Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 -Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 -Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 Baltimore New York 2010-10-25 07:00:00 -25.0 1064 2010-10-25 -Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 Baltimore New York 2010-10-25 07:00:00 106.0 1599 2010-10-25 -Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Baltimore New York 2010-10-25 07:00:00 92.0 1142 2010-10-25 +Baltimore New York 2010-10-26 07:00:00 -22.0 1064 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 123.0 1142 2010-10-26 +Baltimore New York 2010-10-26 07:00:00 90.0 1599 2010-10-26 +Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 +Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 +Baltimore New York 2010-10-28 07:00:00 -14.0 1142 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -14.0 1599 2010-10-28 +Baltimore New York 2010-10-28 07:00:00 -4.0 1064 2010-10-28 +Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 +Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 +Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 +Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 +Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 +Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 +Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 +Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 +Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 +Chicago New York 2010-10-21 07:00:00 -5.0 1610 2010-10-21 +Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 51.0 3198 2010-10-21 +Chicago New York 2010-10-21 07:00:00 53.0 1531 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 +Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 +Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 +Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 +Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 11.0 3085 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 +Chicago New York 2010-10-24 07:00:00 -17.0 1610 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -3.0 3198 2010-10-24 +Chicago New York 2010-10-24 07:00:00 -5.0 1531 2010-10-24 +Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 +Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 +Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 Chicago New York 2010-10-25 07:00:00 43.0 1531 2010-10-25 Chicago New York 2010-10-25 07:00:00 6.0 1610 2010-10-25 -Chicago New York 2010-10-25 07:00:00 -16.0 3198 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 -Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 -Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 -Baltimore New York 2010-10-29 07:00:00 -24.0 1064 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 21.0 1142 2010-10-29 -Baltimore New York 2010-10-29 07:00:00 -2.0 1599 2010-10-29 -Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 +Chicago New York 2010-10-26 07:00:00 -17.0 1610 2010-10-26 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 +Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-26 07:00:00 29.0 1531 2010-10-26 +Chicago New York 2010-10-26 07:00:00 6.0 3198 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 +Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 +Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 +Chicago New York 2010-10-28 07:00:00 -11.0 1531 2010-10-28 +Chicago New York 2010-10-28 07:00:00 -18.0 3198 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 +Chicago New York 2010-10-28 07:00:00 2.0 897 2010-10-28 +Chicago New York 2010-10-28 07:00:00 3.0 1610 2010-10-28 Chicago New York 2010-10-29 07:00:00 -11.0 897 2010-10-29 -Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 Chicago New York 2010-10-29 07:00:00 -18.0 1610 2010-10-29 Chicago New York 2010-10-29 07:00:00 -4.0 3198 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 -Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 -Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 -Baltimore New York 2010-10-20 07:00:00 -30.0 1064 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 23.0 1142 2010-10-20 -Baltimore New York 2010-10-20 07:00:00 6.0 1599 2010-10-20 -Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-20 07:00:00 15.0 1531 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -6.0 1610 2010-10-20 -Chicago New York 2010-10-20 07:00:00 -2.0 3198 2010-10-20 -Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Chicago New York 2010-10-29 07:00:00 15.0 1531 2010-10-29 +Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 +Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 Cleveland New York 2010-10-20 07:00:00 -15.0 3014 2010-10-20 -Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 -Baltimore New York 2010-10-27 07:00:00 -18.0 1064 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 49.0 1142 2010-10-27 -Baltimore New York 2010-10-27 07:00:00 92.0 1599 2010-10-27 -Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 -Chicago New York 2010-10-27 07:00:00 70.0 1531 2010-10-27 -Chicago New York 2010-10-27 07:00:00 8.0 1610 2010-10-27 -Chicago New York 2010-10-27 07:00:00 21.0 3198 2010-10-27 +Cleveland New York 2010-10-20 07:00:00 -8.0 2630 2010-10-20 +Cleveland New York 2010-10-21 07:00:00 29.0 2646 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 3.0 2630 2010-10-21 +Cleveland New York 2010-10-21 07:00:00 72.0 3014 2010-10-21 +Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 +Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 +Cleveland New York 2010-10-23 07:00:00 -21.0 2932 2010-10-23 +Cleveland New York 2010-10-24 07:00:00 -11.0 2630 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -20.0 2646 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 -9.0 3014 2010-10-24 +Cleveland New York 2010-10-24 07:00:00 5.0 2254 2010-10-24 +Cleveland New York 2010-10-25 07:00:00 -4.0 2630 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 42.0 3014 2010-10-25 +Cleveland New York 2010-10-25 07:00:00 81.0 2646 2010-10-25 +Cleveland New York 2010-10-26 07:00:00 -11.0 2662 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 -27.0 2646 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 13.0 3014 2010-10-26 +Cleveland New York 2010-10-26 07:00:00 4.0 2630 2010-10-26 Cleveland New York 2010-10-27 07:00:00 16.0 2630 2010-10-27 Cleveland New York 2010-10-27 07:00:00 27.0 3014 2010-10-27 -Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 -Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 -1.0 1805 2010-10-30 -Baltimore New York 2010-10-30 07:00:00 5.0 3171 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-30 07:00:00 -5.0 3085 2010-10-30 -Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Cleveland New York 2010-10-28 07:00:00 -6.0 2646 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 1.0 3014 2010-10-28 +Cleveland New York 2010-10-28 07:00:00 3.0 2630 2010-10-28 +Cleveland New York 2010-10-29 07:00:00 -12.0 3014 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -19.0 2646 2010-10-29 +Cleveland New York 2010-10-29 07:00:00 -4.0 2630 2010-10-29 Cleveland New York 2010-10-30 07:00:00 -12.0 2932 2010-10-30 -Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 +Cleveland New York 2010-10-30 07:00:00 -23.0 2018 2010-10-30 +Washington New York 2010-10-20 07:00:00 -2.0 7291 2010-10-20 +Washington New York 2010-10-21 07:00:00 22.0 7291 2010-10-21 +Washington New York 2010-10-23 07:00:00 -16.0 7274 2010-10-23 +Washington New York 2010-10-23 07:00:00 -18.0 5917 2010-10-23 +Washington New York 2010-10-23 07:00:00 -21.0 5904 2010-10-23 +Washington New York 2010-10-23 07:00:00 -25.0 5832 2010-10-23 +Washington New York 2010-10-24 07:00:00 -26.0 7282 2010-10-24 +Washington New York 2010-10-25 07:00:00 9.0 7291 2010-10-25 +Washington New York 2010-10-26 07:00:00 4.0 7291 2010-10-26 +Washington New York 2010-10-27 07:00:00 26.0 7291 2010-10-27 +Washington New York 2010-10-28 07:00:00 45.0 7291 2010-10-28 +Washington New York 2010-10-29 07:00:00 1.0 7291 2010-10-29 Washington New York 2010-10-30 07:00:00 -16.0 5917 2010-10-30 -Baltimore New York 2010-10-22 07:00:00 -12.0 1064 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 54.0 1142 2010-10-22 -Baltimore New York 2010-10-22 07:00:00 18.0 1599 2010-10-22 -Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-22 07:00:00 16.0 1531 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -6.0 1610 2010-10-22 -Chicago New York 2010-10-22 07:00:00 -11.0 3198 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 1.0 2630 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -25.0 2646 2010-10-22 -Cleveland New York 2010-10-22 07:00:00 -3.0 3014 2010-10-22 -Baltimore New York 2010-10-31 07:00:00 -1.0 1599 2010-10-31 -Baltimore New York 2010-10-31 07:00:00 -14.0 2571 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -18.0 897 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -4.0 1531 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -22.0 1610 2010-10-31 -Chicago New York 2010-10-31 07:00:00 -15.0 3198 2010-10-31 +Washington New York 2010-10-30 07:00:00 -27.0 5904 2010-10-30 Washington New York 2010-10-31 07:00:00 -18.0 7282 2010-10-31 PREHOOK: query: explain vectorization expression select * from flights_tiny_parquet_partitioned_date sort by fl_num, fl_date limit 25 @@ -3843,7 +3843,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_parquet_partitioned_date - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63269 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -3853,7 +3853,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63269 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: date) sort order: ++ @@ -3861,7 +3861,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63269 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Execution mode: vectorized, llap @@ -3891,13 +3891,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62584 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: date) sort order: ++ @@ -3905,7 +3905,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp), _col3 (type: float) Reducer 3 @@ -3924,19 +3924,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -3982,29 +3982,29 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-3 #### A masked pattern was here #### Baltimore New York 2010-10-23 07:00:00 18.0 272 2010-10-23 Baltimore New York 2010-10-30 07:00:00 14.0 272 2010-10-30 +Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 Chicago New York 2010-10-20 07:00:00 42.0 361 2010-10-20 Chicago New York 2010-10-21 07:00:00 142.0 361 2010-10-21 +Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 Chicago New York 2010-10-22 07:00:00 2.0 361 2010-10-22 +Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 +Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 +Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 +Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-24 07:00:00 10.0 361 2010-10-24 +Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 +Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 Chicago New York 2010-10-25 07:00:00 31.0 361 2010-10-25 +Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 Chicago New York 2010-10-26 07:00:00 12.0 361 2010-10-26 +Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 Chicago New York 2010-10-27 07:00:00 148.0 361 2010-10-27 Chicago New York 2010-10-28 07:00:00 2.0 361 2010-10-28 Chicago New York 2010-10-29 07:00:00 -12.0 361 2010-10-29 -Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 -Chicago New York 2010-10-23 07:00:00 3.0 384 2010-10-23 -Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 -Chicago New York 2010-10-23 07:00:00 32.0 426 2010-10-23 Chicago New York 2010-10-30 07:00:00 -10.0 426 2010-10-30 -Chicago New York 2010-10-23 07:00:00 1.0 650 2010-10-23 Chicago New York 2010-10-30 07:00:00 -5.0 650 2010-10-30 -Chicago New York 2010-10-20 07:00:00 24.0 897 2010-10-20 -Chicago New York 2010-10-21 07:00:00 77.0 897 2010-10-21 -Chicago New York 2010-10-22 07:00:00 24.0 897 2010-10-22 -Chicago New York 2010-10-24 07:00:00 113.0 897 2010-10-24 -Chicago New York 2010-10-25 07:00:00 -1.0 897 2010-10-25 -Chicago New York 2010-10-26 07:00:00 0.0 897 2010-10-26 -Chicago New York 2010-10-27 07:00:00 -11.0 897 2010-10-27 +Chicago New York 2010-10-30 07:00:00 -6.0 384 2010-10-30 +Chicago New York 2010-10-31 07:00:00 -25.0 361 2010-10-31 PREHOOK: query: explain vectorization expression select fl_date, count(*) from flights_tiny_parquet_partitioned_date group by fl_date PREHOOK: type: QUERY @@ -4177,16 +4177,16 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-2 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-30 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### +2010-10-20 11 2010-10-21 12 -2010-10-24 12 -2010-10-26 13 -2010-10-28 12 2010-10-22 11 -2010-10-29 12 -2010-10-20 11 2010-10-23 12 +2010-10-24 12 2010-10-25 12 +2010-10-26 13 2010-10-27 11 +2010-10-28 12 +2010-10-29 12 2010-10-30 11 2010-10-31 8 PREHOOK: query: CREATE TABLE flights_tiny_parquet_partitioned_timestamp ( @@ -4330,142 +4330,142 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### -Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 -Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 -Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 -Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 -Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 -Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 -Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 -Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 -Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 -Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 -Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 -Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 -Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 -Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 -Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 -Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 -Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 -Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 -Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 +Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 +Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 -Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 -Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 -Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 -Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 -Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 -Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 -Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 -Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 +Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 -Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 -Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 -Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 -Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 -Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 PREHOOK: query: select * from flights_tiny_parquet_partitioned_timestamp sort by fl_num, fl_time limit 25 PREHOOK: type: QUERY @@ -4501,29 +4501,29 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010 #### A masked pattern was here #### Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 PREHOOK: query: select fl_time, count(*) from flights_tiny_parquet_partitioned_timestamp group by fl_time PREHOOK: type: QUERY PREHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp @@ -4556,18 +4556,18 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### +2010-10-20 07:00:00 11 2010-10-21 07:00:00 12 -2010-10-24 07:00:00 12 -2010-10-31 07:00:00 8 2010-10-22 07:00:00 11 2010-10-23 07:00:00 12 -2010-10-30 07:00:00 11 -2010-10-20 07:00:00 11 +2010-10-24 07:00:00 12 2010-10-25 07:00:00 12 2010-10-26 07:00:00 13 2010-10-27 07:00:00 11 2010-10-28 07:00:00 12 2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8 PREHOOK: query: explain vectorization expression select * from flights_tiny_parquet_partitioned_timestamp PREHOOK: type: QUERY @@ -4619,7 +4619,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_parquet_partitioned_timestamp - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63189 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -4629,13 +4629,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -4686,146 +4686,146 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-26 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-27 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-28 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 -POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 -#### A masked pattern was here #### -Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 -Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 -Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 -Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 -Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 -Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 -Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-29 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 +POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 +#### A masked pattern was here #### +Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 +Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 +Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 +Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 +Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 +Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 +Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 +Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 +Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 +Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 +Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 +Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 Baltimore New York 2010-10-26 -22.0 1064 2010-10-26 07:00:00 Baltimore New York 2010-10-26 123.0 1142 2010-10-26 07:00:00 Baltimore New York 2010-10-26 90.0 1599 2010-10-26 07:00:00 -Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 -Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 -Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 -Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 -Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 -Baltimore New York 2010-10-21 17.0 1064 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 105.0 1142 2010-10-21 07:00:00 -Baltimore New York 2010-10-21 28.0 1599 2010-10-21 07:00:00 -Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 +Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 +Baltimore New York 2010-10-28 -14.0 1142 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -14.0 1599 2010-10-28 07:00:00 +Baltimore New York 2010-10-28 -4.0 1064 2010-10-28 07:00:00 +Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 +Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 +Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 +Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 +Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 +Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 +Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 +Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 +Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 -5.0 1610 2010-10-21 07:00:00 +Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 Chicago New York 2010-10-21 51.0 3198 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 -Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 -Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 -Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 -10.0 1805 2010-10-23 07:00:00 -Baltimore New York 2010-10-23 6.0 3171 2010-10-23 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 +Chicago New York 2010-10-21 53.0 1531 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 +Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 +Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 +Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 +Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-23 11.0 3085 2010-10-23 07:00:00 -Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 -Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 -Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 -Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 -Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 -Baltimore New York 2010-10-24 12.0 1599 2010-10-24 07:00:00 -Baltimore New York 2010-10-24 20.0 2571 2010-10-24 07:00:00 -Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-24 -17.0 1610 2010-10-24 07:00:00 Chicago New York 2010-10-24 -3.0 3198 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 -Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 -Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 -Baltimore New York 2010-10-25 -25.0 1064 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 92.0 1142 2010-10-25 07:00:00 -Baltimore New York 2010-10-25 106.0 1599 2010-10-25 07:00:00 -Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-24 -5.0 1531 2010-10-24 07:00:00 +Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 +Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 +Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 Chicago New York 2010-10-25 43.0 1531 2010-10-25 07:00:00 Chicago New York 2010-10-25 6.0 1610 2010-10-25 07:00:00 -Chicago New York 2010-10-25 -16.0 3198 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 -Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 -Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 -Baltimore New York 2010-10-29 -24.0 1064 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 21.0 1142 2010-10-29 07:00:00 -Baltimore New York 2010-10-29 -2.0 1599 2010-10-29 07:00:00 -Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 +Chicago New York 2010-10-26 -17.0 1610 2010-10-26 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 +Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-26 29.0 1531 2010-10-26 07:00:00 +Chicago New York 2010-10-26 6.0 3198 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 +Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 +Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 +Chicago New York 2010-10-28 -11.0 1531 2010-10-28 07:00:00 +Chicago New York 2010-10-28 -18.0 3198 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 +Chicago New York 2010-10-28 2.0 897 2010-10-28 07:00:00 +Chicago New York 2010-10-28 3.0 1610 2010-10-28 07:00:00 Chicago New York 2010-10-29 -11.0 897 2010-10-29 07:00:00 -Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 Chicago New York 2010-10-29 -18.0 1610 2010-10-29 07:00:00 Chicago New York 2010-10-29 -4.0 3198 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 -Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 -Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 -Baltimore New York 2010-10-20 -30.0 1064 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 23.0 1142 2010-10-20 07:00:00 -Baltimore New York 2010-10-20 6.0 1599 2010-10-20 07:00:00 -Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-20 15.0 1531 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -6.0 1610 2010-10-20 07:00:00 -Chicago New York 2010-10-20 -2.0 3198 2010-10-20 07:00:00 -Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Chicago New York 2010-10-29 15.0 1531 2010-10-29 07:00:00 +Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 +Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 Cleveland New York 2010-10-20 -15.0 3014 2010-10-20 07:00:00 -Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 -Baltimore New York 2010-10-27 -18.0 1064 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 49.0 1142 2010-10-27 07:00:00 -Baltimore New York 2010-10-27 92.0 1599 2010-10-27 07:00:00 -Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 -Chicago New York 2010-10-27 70.0 1531 2010-10-27 07:00:00 -Chicago New York 2010-10-27 8.0 1610 2010-10-27 07:00:00 -Chicago New York 2010-10-27 21.0 3198 2010-10-27 07:00:00 +Cleveland New York 2010-10-20 -8.0 2630 2010-10-20 07:00:00 +Cleveland New York 2010-10-21 29.0 2646 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 3.0 2630 2010-10-21 07:00:00 +Cleveland New York 2010-10-21 72.0 3014 2010-10-21 07:00:00 +Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 +Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 +Cleveland New York 2010-10-23 -21.0 2932 2010-10-23 07:00:00 +Cleveland New York 2010-10-24 -11.0 2630 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -20.0 2646 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 -9.0 3014 2010-10-24 07:00:00 +Cleveland New York 2010-10-24 5.0 2254 2010-10-24 07:00:00 +Cleveland New York 2010-10-25 -4.0 2630 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 42.0 3014 2010-10-25 07:00:00 +Cleveland New York 2010-10-25 81.0 2646 2010-10-25 07:00:00 +Cleveland New York 2010-10-26 -11.0 2662 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 -27.0 2646 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 13.0 3014 2010-10-26 07:00:00 +Cleveland New York 2010-10-26 4.0 2630 2010-10-26 07:00:00 Cleveland New York 2010-10-27 16.0 2630 2010-10-27 07:00:00 Cleveland New York 2010-10-27 27.0 3014 2010-10-27 07:00:00 -Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 -Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 -1.0 1805 2010-10-30 07:00:00 -Baltimore New York 2010-10-30 5.0 3171 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-30 -5.0 3085 2010-10-30 07:00:00 -Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Cleveland New York 2010-10-28 -6.0 2646 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 1.0 3014 2010-10-28 07:00:00 +Cleveland New York 2010-10-28 3.0 2630 2010-10-28 07:00:00 +Cleveland New York 2010-10-29 -12.0 3014 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -19.0 2646 2010-10-29 07:00:00 +Cleveland New York 2010-10-29 -4.0 2630 2010-10-29 07:00:00 Cleveland New York 2010-10-30 -12.0 2932 2010-10-30 07:00:00 -Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 +Cleveland New York 2010-10-30 -23.0 2018 2010-10-30 07:00:00 +Washington New York 2010-10-20 -2.0 7291 2010-10-20 07:00:00 +Washington New York 2010-10-21 22.0 7291 2010-10-21 07:00:00 +Washington New York 2010-10-23 -16.0 7274 2010-10-23 07:00:00 +Washington New York 2010-10-23 -18.0 5917 2010-10-23 07:00:00 +Washington New York 2010-10-23 -21.0 5904 2010-10-23 07:00:00 +Washington New York 2010-10-23 -25.0 5832 2010-10-23 07:00:00 +Washington New York 2010-10-24 -26.0 7282 2010-10-24 07:00:00 +Washington New York 2010-10-25 9.0 7291 2010-10-25 07:00:00 +Washington New York 2010-10-26 4.0 7291 2010-10-26 07:00:00 +Washington New York 2010-10-27 26.0 7291 2010-10-27 07:00:00 +Washington New York 2010-10-28 45.0 7291 2010-10-28 07:00:00 +Washington New York 2010-10-29 1.0 7291 2010-10-29 07:00:00 Washington New York 2010-10-30 -16.0 5917 2010-10-30 07:00:00 -Baltimore New York 2010-10-22 -12.0 1064 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 54.0 1142 2010-10-22 07:00:00 -Baltimore New York 2010-10-22 18.0 1599 2010-10-22 07:00:00 -Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-22 16.0 1531 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -6.0 1610 2010-10-22 07:00:00 -Chicago New York 2010-10-22 -11.0 3198 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 1.0 2630 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -25.0 2646 2010-10-22 07:00:00 -Cleveland New York 2010-10-22 -3.0 3014 2010-10-22 07:00:00 -Baltimore New York 2010-10-31 -1.0 1599 2010-10-31 07:00:00 -Baltimore New York 2010-10-31 -14.0 2571 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -18.0 897 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -4.0 1531 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -22.0 1610 2010-10-31 07:00:00 -Chicago New York 2010-10-31 -15.0 3198 2010-10-31 07:00:00 +Washington New York 2010-10-30 -27.0 5904 2010-10-30 07:00:00 Washington New York 2010-10-31 -18.0 7282 2010-10-31 07:00:00 PREHOOK: query: explain vectorization expression select * from flights_tiny_parquet_partitioned_timestamp sort by fl_num, fl_time limit 25 @@ -4882,7 +4882,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: flights_tiny_parquet_partitioned_timestamp - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63189 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true Select Operator @@ -4892,7 +4892,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4, 5] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63189 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: timestamp) sort order: ++ @@ -4900,7 +4900,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 63189 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Execution mode: vectorized, llap @@ -4930,13 +4930,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 137 Data size: 39593 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 137 Data size: 62504 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col4 (type: int), _col5 (type: timestamp) sort order: ++ @@ -4944,7 +4944,7 @@ STAGE PLANS: className: VectorReduceSinkObjectHashOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: date), _col3 (type: float) Reducer 3 @@ -4963,19 +4963,19 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 5, 0, 1] - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL Limit Number of rows: 25 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 25 Data size: 7225 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 25 Data size: 11800 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -5021,29 +5021,29 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010 #### A masked pattern was here #### Baltimore New York 2010-10-23 18.0 272 2010-10-23 07:00:00 Baltimore New York 2010-10-30 14.0 272 2010-10-30 07:00:00 +Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 Chicago New York 2010-10-20 42.0 361 2010-10-20 07:00:00 Chicago New York 2010-10-21 142.0 361 2010-10-21 07:00:00 +Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 Chicago New York 2010-10-22 2.0 361 2010-10-22 07:00:00 +Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 +Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 +Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 +Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-24 10.0 361 2010-10-24 07:00:00 +Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 +Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 Chicago New York 2010-10-25 31.0 361 2010-10-25 07:00:00 +Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 Chicago New York 2010-10-26 12.0 361 2010-10-26 07:00:00 +Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 Chicago New York 2010-10-27 148.0 361 2010-10-27 07:00:00 Chicago New York 2010-10-28 2.0 361 2010-10-28 07:00:00 Chicago New York 2010-10-29 -12.0 361 2010-10-29 07:00:00 -Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 -Chicago New York 2010-10-23 3.0 384 2010-10-23 07:00:00 -Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 -Chicago New York 2010-10-23 32.0 426 2010-10-23 07:00:00 Chicago New York 2010-10-30 -10.0 426 2010-10-30 07:00:00 -Chicago New York 2010-10-23 1.0 650 2010-10-23 07:00:00 Chicago New York 2010-10-30 -5.0 650 2010-10-30 07:00:00 -Chicago New York 2010-10-20 24.0 897 2010-10-20 07:00:00 -Chicago New York 2010-10-21 77.0 897 2010-10-21 07:00:00 -Chicago New York 2010-10-22 24.0 897 2010-10-22 07:00:00 -Chicago New York 2010-10-24 113.0 897 2010-10-24 07:00:00 -Chicago New York 2010-10-25 -1.0 897 2010-10-25 07:00:00 -Chicago New York 2010-10-26 0.0 897 2010-10-26 07:00:00 -Chicago New York 2010-10-27 -11.0 897 2010-10-27 07:00:00 +Chicago New York 2010-10-30 -6.0 384 2010-10-30 07:00:00 +Chicago New York 2010-10-31 -25.0 361 2010-10-31 07:00:00 PREHOOK: query: explain vectorization expression select fl_time, count(*) from flights_tiny_parquet_partitioned_timestamp group by fl_time PREHOOK: type: QUERY @@ -5216,15 +5216,15 @@ POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-30 07%3A00%3A00 POSTHOOK: Input: default@flights_tiny_parquet_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### +2010-10-20 07:00:00 11 2010-10-21 07:00:00 12 -2010-10-24 07:00:00 12 -2010-10-31 07:00:00 8 2010-10-22 07:00:00 11 2010-10-23 07:00:00 12 -2010-10-30 07:00:00 11 -2010-10-20 07:00:00 11 +2010-10-24 07:00:00 12 2010-10-25 07:00:00 12 2010-10-26 07:00:00 13 2010-10-27 07:00:00 11 2010-10-28 07:00:00 12 2010-10-29 07:00:00 12 +2010-10-30 07:00:00 11 +2010-10-31 07:00:00 8