diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index cc6239c313..f3480ab2fa 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2216,6 +2216,16 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "When enabled dynamic partitioning column will be globally sorted.\n" + "This way we can keep only one record writer open for each partition value\n" + "in the reducer thereby reducing the memory pressure on reducers."), + HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD("hive.optimize.sort.dynamic.partition.threshold", 0, + "When enabled dynamic partitioning column will be globally sorted.\n" + + "This way we can keep only one record writer open for each partition value\n" + + "in the reducer thereby reducing the memory pressure on reducers.\n" + + "This config has following possible values: \n" + + "\t-1 - This completely disables the optimization. \n" + + "\t1 - This always enable the optimization. \n" + + "\t0 - This makes the optimization a cost based decision. \n" + + "Setting it to any other positive integer will make Hive use this as threshold for number of writers."), + HIVESAMPLINGFORORDERBY("hive.optimize.sampling.orderby", false, "Uses sampling on order-by clause for parallel execution."), HIVESAMPLINGNUMBERFORORDERBY("hive.optimize.sampling.orderby.number", 1000, "Total number of samples to be obtained."), diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out index b42f966369..80dbbee5ba 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out @@ -84,7 +84,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -107,13 +108,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _bucket_number (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col1 (type: string) + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -159,16 +159,15 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _bucket_number - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### - Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -195,6 +194,34 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: id, key + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(id, 'hll') + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Move Operator @@ -236,6 +263,83 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: struct) + auto parallelism: false + Execution mode: vectorized + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out index 13d64fb9fa..a8cdc8fe05 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out @@ -102,7 +102,8 @@ POSTHOOK: Input: _dummy_database@_dummy_table STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-0, Stage-3 + Stage-3 depends on stages: Stage-1 STAGE PLANS: Stage: Stage-1 @@ -125,13 +126,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: string), _bucket_number (type: string) - null sort order: aa - sort order: ++ - Map-reduce partition columns: _col1 (type: string) + null sort order: + sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE tag: -1 - value expressions: _col0 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -177,16 +177,15 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _bucket_number - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false GlobalTableId: 1 directory: ### BLOBSTORE_STAGING_PATH ### - Dp Sort State: PARTITION_BUCKET_SORTED NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE Stats Publishing Key Prefix: ### BLOBSTORE_STAGING_PATH ### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -213,6 +212,34 @@ STAGE PLANS: TotalFiles: 1 GatherStats: true MultiFileSpray: false + Select Operator + expressions: _col0 (type: int), _col1 (type: string) + outputColumnNames: id, key + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(id, 'hll') + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false Stage: Stage-0 Move Operator @@ -254,6 +281,83 @@ STAGE PLANS: Table: default.table1 Is Table Level Stats: false + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 608 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col1 (type: struct) + auto parallelism: false + Execution mode: vectorized + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10002 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1 + columns.types string,struct + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: struct), _col0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 624 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1 + columns.types struct:string + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + PREHOOK: query: DROP TABLE table1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@table1 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out index 7b2561358a..826fae9d42 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -200 +0 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -50 +0 PREHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -100 +0 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -300 +0 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -350 +0 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -400 +50 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -350 +50 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out index 1201ce2107..bb63070503 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -200 +0 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -300 +100 PREHOOK: query: INSERT INTO TABLE orc_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -400 +200 PREHOOK: query: ALTER TABLE orc_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM orc_events POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_events #### A masked pattern was here #### -500 +300 PREHOOK: query: INSERT OVERWRITE TABLE orc_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out index 0931e3d557..7758dc9d53 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -200 +0 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -50 +0 PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@parquet_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -100 +0 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -300 +0 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -350 +0 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -400 +50 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -350 +50 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out index 15ae3d91a7..0ccd0e4345 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -200 +0 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -300 +100 PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -400 +200 PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -303,7 +303,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM parquet_events POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_events #### A masked pattern was here #### -500 +300 PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) SELECT * FROM src_events PREHOOK: type: QUERY diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out index 24fc525804..340791aa7f 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -200 +0 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -152,7 +152,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20120921 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -50 +0 PREHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -161,7 +161,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events WHERE run_date=20121121 POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -100 +0 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201211, game_id, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -200,7 +200,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -300 +0 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name) SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 @@ -229,7 +229,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -350 +0 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -258,7 +258,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -400 +50 PREHOOK: query: INSERT OVERWRITE TABLE rcfile_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' @@ -287,4 +287,4 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -350 +50 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out index 6bcfe41262..160842236b 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out @@ -143,7 +143,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -200 +0 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS @@ -193,7 +193,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -300 +100 PREHOOK: query: INSERT INTO TABLE rcfile_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events WHERE SUBSTR(run_date,1,6)='201211' @@ -232,7 +232,7 @@ POSTHOOK: query: SELECT COUNT(*) FROM rcfile_events POSTHOOK: type: QUERY POSTHOOK: Input: default@rcfile_events #### A masked pattern was here #### -400 +200 PREHOOK: query: ALTER TABLE rcfile_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') #### A masked pattern was here #### PREHOOK: type: ALTERTABLE_ADDPARTS diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index 71f73802c2..25e9cd0482 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -115,12 +115,7 @@ public void initialize(HiveConf hiveConf) { transformations.add(new ConstantPropagate()); } - if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) && - HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") && - HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) && - !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { - transformations.add(new SortedDynPartitionOptimizer()); - } + transformations.add(new SortedDynPartitionTimeGranularityOptimizer()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index 2dc2351793..02f93d0923 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.Stack; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -55,6 +56,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ColStatistics; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -69,6 +71,7 @@ import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.orc.OrcConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -174,13 +177,18 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // unlink connection between FS and its parent fsParent = fsOp.getParentOperators().get(0); + DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx(); + List partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema()); + if(!shouldDo(partitionPositions, fsParent)) { + return null; + } + fsParent.getChildOperators().clear(); - DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx(); - int numBuckets = destTable.getNumBuckets(); // if enforce bucketing/sorting is disabled numBuckets will not be set. // set the number of buckets here to ensure creation of empty buckets + int numBuckets = destTable.getNumBuckets(); dpCtx.setNumBuckets(numBuckets); // Get the positions for partition, bucket and sort columns @@ -232,7 +240,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, for (int i : sortPositions) LOG.debug("sort position " + i); for (int i : sortOrder) LOG.debug("sort order " + i); for (int i : sortNullOrder) LOG.debug("sort null order " + i); - List partitionPositions = getPartitionPositions(dpCtx, fsParent.getSchema()); // update file sink descriptor fsOp.getConf().setMultiFileSpray(false); @@ -651,6 +658,60 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, return cols; } - } + // the idea is to estimate how many number of writers this insert can spun up. + // Writers are proportional to number of partitions being inserted i.e cardinality of the partition columns + // if these writers are less than number of writers allowed within the memory pool (estimated) we go ahead with + // adding extra RS + // The way max number of writers allowed are computed based on + // (executor/container memory) * (percentage of memory taken by orc) + // and dividing that by max memory (stripe size) taken by a single writer. + //TODO: take number of buckets into account + private boolean shouldDo(List partitionPos, Operator fsParent) { + int threshold = HiveConf.getIntVar(this.parseCtx.getConf(), + HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITIONTHRESHOLD); + long MAX_WRITERS = -1; + switch(threshold) { + case -1: + return false; + case 0: + break; + case 1: + return true; + default: + MAX_WRITERS = threshold; + break; + } + List colStats = fsParent.getStatistics().getColumnStats(); + if(colStats == null || colStats.isEmpty()) { + return false; + } + long partCardinality = 1; + + // compute cardinality for partition columns + for(Integer idx:partitionPos) { + ColumnInfo ci = fsParent.getSchema().getSignature().get(idx); + ColStatistics partStats = fsParent.getStatistics().getColumnStatisticsFromColName(ci.getInternalName()); + if(partStats == null) { + // statistics for this partition are for some reason not available + return false; + } + partCardinality = partCardinality * partStats.getCountDistint(); + } + + if(MAX_WRITERS < 0) { + double orcMemPool = this.parseCtx.getConf().getDouble(OrcConf.MEMORY_POOL.getHiveConfName(), + (Double)OrcConf.MEMORY_POOL.getDefaultValue()); + long orcStripSize = this.parseCtx.getConf().getLong(OrcConf.STRIPE_SIZE.getHiveConfName(), + (Long)OrcConf.STRIPE_SIZE.getDefaultValue()); + long executorMem = 4000000000L; + MAX_WRITERS = (long)(executorMem * orcMemPool)/orcStripSize; + + } + if(partCardinality <= MAX_WRITERS) { + return false; + } + return true; + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index fdc963506c..272d4864fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -72,16 +72,9 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; +import org.apache.hadoop.hive.ql.optimizer.*; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; -import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin; -import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization; -import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc; -import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc; -import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize; -import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism; -import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer; -import org.apache.hadoop.hive.ql.optimizer.TopNKeyProcessor; +import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkDeDuplication; import org.apache.hadoop.hive.ql.optimizer.correlation.ReduceSinkJoinDeDuplication; import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits; import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer; @@ -170,6 +163,27 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // Update bucketing version of ReduceSinkOp if needed updateBucketingVersionForUpgrade(procCtx); + // run Sorted dynamic partition optimization + if(HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.DYNAMICPARTITIONING) && + HiveConf.getVar(procCtx.conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") && + !HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + new SortedDynPartitionOptimizer().transform(procCtx.parseContext); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Sorted dynamic partition optimization"); + } + + if(HiveConf.getBoolVar(procCtx.conf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION) + || procCtx.parseContext.hasAcidWrite()) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + // Dynamic sort partition adds an extra RS therefore need to de-dup + new ReduceSinkDeDuplication().transform(procCtx.parseContext); + // there is an issue with dedup logic wherein SELECT is created with wrong columns + // NonBlockingOpDeDupProc fixes that + // (kind of hackish, the issue in de-dup should be fixed but it needs more investigation) + new NonBlockingOpDeDupProc().transform(procCtx.parseContext); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Reduce Sink de-duplication"); + } + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // run the optimizations that use stats for optimization runStatsDependentOptimizations(procCtx, inputs, outputs); diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q index 3c2918fdca..d5148560aa 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q @@ -8,8 +8,7 @@ set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; set hive.exec.dynamic.partition.mode=nonstrict; set hive.vectorized.execution.enabled=true; - - +set hive.optimize.sort.dynamic.partition.threshold=1; create table over1k_n1( t tinyint, diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q index 436c0edc77..1296ba88d5 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q @@ -7,8 +7,7 @@ set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; set hive.exec.dynamic.partition.mode=nonstrict; - - +set hive.optimize.sort.dynamic.partition.threshold=1; create table over1k_n3( t tinyint, @@ -209,3 +208,40 @@ insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from ov insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k_n3 where i=100 and t=27 and s="foo"; select sum(hash(*)) from over1k_part3; + +drop table over1k_n3; +create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|'; + +load data local inpath '../../data/files/over1k' into table over1k_n3; + +analyze table over1k_n3 compute statistics for columns; +set hive.stats.fetch.column.stats=true; +set hive.optimize.sort.dynamic.partition=true; + +-- default hive should do cost based and add extra RS +set hive.optimize.sort.dynamic.partition.threshold=0; +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27; + +-- default but shouldn't add extra RS +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10; + +-- disable +set hive.optimize.sort.dynamic.partition.threshold=-1; +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27; + +-- enable, will add extra RS +set hive.optimize.sort.dynamic.partition.threshold=1; +explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10; diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q index 6dfb51a7da..78d08459b5 100644 --- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q +++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization2.q @@ -6,8 +6,7 @@ set hive.exec.dynamic.partition=true; set hive.exec.max.dynamic.partitions=1000; set hive.exec.max.dynamic.partitions.pernode=1000; set hive.exec.dynamic.partition.mode=nonstrict; - - +set hive.optimize.sort.dynamic.partition.threshold=1; -- SORT_QUERY_RESULTS diff --git a/ql/src/test/queries/clientpositive/llap_acid.q b/ql/src/test/queries/clientpositive/llap_acid.q index 754461d8ea..24f16c360c 100644 --- a/ql/src/test/queries/clientpositive/llap_acid.q +++ b/ql/src/test/queries/clientpositive/llap_acid.q @@ -13,6 +13,9 @@ set hive.exec.dynamic.partition.mode=nonstrict; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; + DROP TABLE orc_llap_n1; CREATE TABLE orc_llap_n1 ( diff --git a/ql/src/test/queries/clientpositive/llap_acid_fast.q b/ql/src/test/queries/clientpositive/llap_acid_fast.q index 1342efb71d..d04dc728ff 100644 --- a/ql/src/test/queries/clientpositive/llap_acid_fast.q +++ b/ql/src/test/queries/clientpositive/llap_acid_fast.q @@ -13,6 +13,9 @@ set hive.exec.dynamic.partition.mode=nonstrict; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; + DROP TABLE orc_llap_acid_fast; CREATE TABLE orc_llap_acid_fast ( diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q index b540b6e635..4e0d3ebf82 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update.q @@ -17,6 +17,8 @@ set hive.exec.dynamic.partition.mode=nonstrict; set hive.metastore.disallow.incompatible.col.type.changes=true; set hive.default.fileformat=orc; set hive.llap.io.enabled=false; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; -- SORT_QUERY_RESULTS -- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q index 9dfdf97113..187896404b 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_part_update_llap_io.q @@ -18,6 +18,8 @@ set hive.metastore.disallow.incompatible.col.type.changes=true; set hive.default.fileformat=orc; set hive.llap.io.enabled=true; set hive.llap.io.encode.enabled=true; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; -- SORT_QUERY_RESULTS -- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update.q index 6cbb1c1dad..011f788d1d 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update.q @@ -15,6 +15,8 @@ set hive.exec.dynamic.partition.mode=nonstrict; set hive.metastore.disallow.incompatible.col.type.changes=true; set hive.default.fileformat=orc; set hive.llap.io.enabled=false; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; -- SORT_QUERY_RESULTS -- diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update_llap_io.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update_llap_io.q index 9c8bdda7ee..bb5a30c5fb 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update_llap_io.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acidvec_part_update_llap_io.q @@ -16,6 +16,8 @@ set hive.metastore.disallow.incompatible.col.type.changes=true; set hive.default.fileformat=orc; set hive.llap.io.enabled=true; set hive.llap.io.encode.enabled=true; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; -- SORT_QUERY_RESULTS -- diff --git a/ql/src/test/queries/clientpositive/update_all_partitioned.q b/ql/src/test/queries/clientpositive/update_all_partitioned.q index 0b34fc2bcd..d7a81c5093 100644 --- a/ql/src/test/queries/clientpositive/update_all_partitioned.q +++ b/ql/src/test/queries/clientpositive/update_all_partitioned.q @@ -2,6 +2,9 @@ set hive.mapred.mode=nonstrict; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +-- BUG: HIVE-20719 +set hive.vectorized.execution.enabled=false; + create table acid_uap(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); diff --git a/ql/src/test/results/clientpositive/acid_table_stats.q.out b/ql/src/test/results/clientpositive/acid_table_stats.q.out index e1af88ad81..e06624c32b 100644 --- a/ql/src/test/results/clientpositive/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/acid_table_stats.q.out @@ -97,7 +97,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4063 + totalSize 4542 #### A masked pattern was here #### # Storage Information @@ -184,7 +184,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4063 + totalSize 4542 #### A masked pattern was here #### # Storage Information @@ -235,7 +235,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4063 + totalSize 4542 #### A masked pattern was here #### # Storage Information @@ -331,7 +331,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8126 + totalSize 9085 #### A masked pattern was here #### # Storage Information @@ -380,7 +380,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8126 + totalSize 9085 #### A masked pattern was here #### # Storage Information @@ -455,11 +455,11 @@ STAGE PLANS: TableScan alias: acid filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: key - Statistics: Num rows: 2000 Data size: 81260 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2000 Data size: 90850 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: max(key) mode: hash @@ -591,7 +591,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 176000 - totalSize 3008 + totalSize 3485 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out index 20f88bdb70..5bc944271d 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization_acid2.q.out @@ -31,8 +31,10 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - Stage-2 depends on stages: Stage-0 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0, Stage-4 + Stage-4 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -46,26 +48,62 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col2 (type: string), _col3 (type: string), _bucket_number (type: string), _col1 (type: string) - sort order: ++++ - Map-reduce partition columns: _col2 (type: string), _col3 (type: string) + key expressions: _col1 (type: string) + sort order: + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: vectorized Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + sort order: + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + Execution mode: vectorized + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), UDFToInteger(_col3) (type: int) + outputColumnNames: key, value, ds, hr + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') + keys: ds (type: string), hr (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-0 Move Operator @@ -80,7 +118,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.non_acid - Stage: Stage-2 + Stage: Stage-3 Stats Work Basic Stats Work: Column Stats Desc: @@ -88,3 +126,33 @@ STAGE PLANS: Column Types: string, string Table: default.non_acid + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct), _col3 (type: struct) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out index 0414812ca9..1d629abc21 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_16.q.out @@ -223,16 +223,16 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: pri BETWEEN 1 AND 2 (type: boolean) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: bigint) sort order: + Map-reduce partition columns: _col0 (type: bigint) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) Execution mode: vectorized, llap LLAP IO: no inputs @@ -246,14 +246,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -371,11 +371,11 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: pri BETWEEN 1 AND 2 (type: boolean) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -392,14 +392,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -446,30 +446,11 @@ POSTHOOK: Input: default@bucket_small_n17 POSTHOOK: Input: default@bucket_small_n17@pri=1 POSTHOOK: Input: default@bucket_small_n17@pri=2 #### A masked pattern was here #### -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -103 val_103 val_103 day1 1 -103 val_103 val_103 day1 1 -103 val_103 val_103 day1 1 -103 val_103 val_103 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -172 val_172 val_172 day1 1 -172 val_172 val_172 day1 1 -172 val_172 val_172 day1 1 -172 val_172 val_172 day1 1 -374 val_374 val_374 day1 1 -374 val_374 val_374 day1 1 +0 val_0 NULL day1 1 +103 val_103 NULL day1 1 +169 val_169 NULL day1 1 +172 val_172 NULL day1 1 +374 val_374 NULL day1 1 PREHOOK: query: drop table bucket_big_n17 PREHOOK: type: DROPTABLE PREHOOK: Input: default@bucket_big_n17 @@ -627,11 +608,11 @@ STAGE PLANS: TableScan alias: bucket_small_n17 filterExpr: pri BETWEEN 1 AND 2 (type: boolean) - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: bigint), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 236 Data size: 23364 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 236 Data size: 45636 Basic stats: COMPLETE Column stats: NONE Map Operator Tree: TableScan alias: bucket_big_n17 @@ -648,14 +629,14 @@ STAGE PLANS: 0 _col0 (type: bigint) 1 _col0 (type: bigint) outputColumnNames: _col0, _col1, _col3 - Statistics: Num rows: 20 Data size: 3436 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint), _col1 (type: string), _col3 (type: string), 'day1' (type: string), 1 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 20 Data size: 5276 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 259 Data size: 50199 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -702,27 +683,8 @@ POSTHOOK: Input: default@bucket_small_n17 POSTHOOK: Input: default@bucket_small_n17@pri=1 POSTHOOK: Input: default@bucket_small_n17@pri=2 #### A masked pattern was here #### -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -0 val_0 val_0 day1 1 -103 val_103 val_103 day1 1 -103 val_103 val_103 day1 1 -103 val_103 val_103 day1 1 -103 val_103 val_103 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -169 val_169 val_169 day1 1 -172 val_172 val_172 day1 1 -172 val_172 val_172 day1 1 -172 val_172 val_172 day1 1 -172 val_172 val_172 day1 1 -374 val_374 val_374 day1 1 -374 val_374 val_374 day1 1 +0 val_0 NULL day1 1 +103 val_103 NULL day1 1 +169 val_169 NULL day1 1 +172 val_172 NULL day1 1 +374 val_374 NULL day1 1 diff --git a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out b/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out index 4ca60ba5ce..bb18262232 100644 --- a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out @@ -17,25 +17,25 @@ PREHOOK: Output: default@src2_n5 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 57 - CREATED_FILES: 61 + CREATED_FILES: 57 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 57 RECORDS_OUT_1_default.src2_n5: 84 - RECORDS_OUT_INTERMEDIATE_Map_1: 57 + RECORDS_OUT_INTERMEDIATE_Map_1: 84 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 84 - RECORDS_OUT_OPERATOR_FS_11: 57 - RECORDS_OUT_OPERATOR_FS_4: 84 - RECORDS_OUT_OPERATOR_GBY_7: 57 - RECORDS_OUT_OPERATOR_GBY_9: 57 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 84 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 57 - RECORDS_OUT_OPERATOR_SEL_10: 57 - RECORDS_OUT_OPERATOR_SEL_2: 84 - RECORDS_OUT_OPERATOR_SEL_6: 84 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 84 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 57 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 84 +Stage-1 LLAP IO COUNTERS: + CACHE_MISS_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -48,25 +48,25 @@ PREHOOK: Output: default@src2_n5 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 64 - CREATED_FILES: 125 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n5: 189 - RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Map_1: 189 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 189 - RECORDS_OUT_OPERATOR_FS_11: 121 - RECORDS_OUT_OPERATOR_FS_4: 189 - RECORDS_OUT_OPERATOR_GBY_7: 121 - RECORDS_OUT_OPERATOR_GBY_9: 121 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 189 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 121 - RECORDS_OUT_OPERATOR_SEL_10: 121 - RECORDS_OUT_OPERATOR_SEL_2: 189 - RECORDS_OUT_OPERATOR_SEL_6: 189 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 189 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 121 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -87,25 +87,25 @@ PREHOOK: Output: default@src2_n5 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 121 - CREATED_FILES: 125 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n5: 189 - RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Map_1: 189 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 189 - RECORDS_OUT_OPERATOR_FS_11: 121 - RECORDS_OUT_OPERATOR_FS_4: 189 - RECORDS_OUT_OPERATOR_GBY_7: 121 - RECORDS_OUT_OPERATOR_GBY_9: 121 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 189 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 121 - RECORDS_OUT_OPERATOR_SEL_10: 121 - RECORDS_OUT_OPERATOR_SEL_2: 189 - RECORDS_OUT_OPERATOR_SEL_6: 189 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 189 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 121 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -118,25 +118,25 @@ PREHOOK: Output: default@src2_n5 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 63 - CREATED_FILES: 188 + CREATED_FILES: 184 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2_n5: 292 - RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Map_1: 292 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 292 - RECORDS_OUT_OPERATOR_FS_11: 184 - RECORDS_OUT_OPERATOR_FS_4: 292 - RECORDS_OUT_OPERATOR_GBY_7: 184 - RECORDS_OUT_OPERATOR_GBY_9: 184 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 292 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 184 - RECORDS_OUT_OPERATOR_SEL_10: 184 - RECORDS_OUT_OPERATOR_SEL_2: 292 - RECORDS_OUT_OPERATOR_SEL_6: 292 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 292 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 184 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 292 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -166,36 +166,32 @@ PREHOOK: Output: default@src3_n1 Stage-2 FILE SYSTEM COUNTERS: Stage-2 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 121 - CREATED_FILES: 129 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n5: 84 RECORDS_OUT_2_default.src3_n1: 105 - RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Map_1: 189 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_23: 84 - RECORDS_OUT_OPERATOR_FIL_24: 105 - RECORDS_OUT_OPERATOR_FS_11: 57 - RECORDS_OUT_OPERATOR_FS_15: 105 - RECORDS_OUT_OPERATOR_FS_22: 64 - RECORDS_OUT_OPERATOR_FS_4: 84 - RECORDS_OUT_OPERATOR_GBY_18: 64 - RECORDS_OUT_OPERATOR_GBY_20: 64 - RECORDS_OUT_OPERATOR_GBY_7: 57 - RECORDS_OUT_OPERATOR_GBY_9: 57 + RECORDS_OUT_OPERATOR_FIL_29: 1 + RECORDS_OUT_OPERATOR_FIL_30: 1 + RECORDS_OUT_OPERATOR_FS_36: 84 + RECORDS_OUT_OPERATOR_FS_38: 105 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_19: 64 - RECORDS_OUT_OPERATOR_RS_8: 57 - RECORDS_OUT_OPERATOR_SEL_10: 57 - RECORDS_OUT_OPERATOR_SEL_13: 105 - RECORDS_OUT_OPERATOR_SEL_17: 105 - RECORDS_OUT_OPERATOR_SEL_2: 84 - RECORDS_OUT_OPERATOR_SEL_21: 64 - RECORDS_OUT_OPERATOR_SEL_6: 84 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_33: 84 + RECORDS_OUT_OPERATOR_RS_34: 105 + RECORDS_OUT_OPERATOR_SEL_31: 1 + RECORDS_OUT_OPERATOR_SEL_32: 1 + RECORDS_OUT_OPERATOR_SEL_35: 57 + RECORDS_OUT_OPERATOR_SEL_37: 64 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-2 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -211,36 +207,32 @@ PREHOOK: Output: default@src3_n1 Stage-2 FILE SYSTEM COUNTERS: Stage-2 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 63 - CREATED_FILES: 192 + CREATED_FILES: 184 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2_n5: 84 RECORDS_OUT_2_default.src3_n1: 208 - RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Map_1: 292 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_23: 84 - RECORDS_OUT_OPERATOR_FIL_24: 208 - RECORDS_OUT_OPERATOR_FS_11: 57 - RECORDS_OUT_OPERATOR_FS_15: 208 - RECORDS_OUT_OPERATOR_FS_22: 127 - RECORDS_OUT_OPERATOR_FS_4: 84 - RECORDS_OUT_OPERATOR_GBY_18: 127 - RECORDS_OUT_OPERATOR_GBY_20: 127 - RECORDS_OUT_OPERATOR_GBY_7: 57 - RECORDS_OUT_OPERATOR_GBY_9: 57 + RECORDS_OUT_OPERATOR_FIL_29: 1 + RECORDS_OUT_OPERATOR_FIL_30: 1 + RECORDS_OUT_OPERATOR_FS_36: 84 + RECORDS_OUT_OPERATOR_FS_38: 208 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_19: 127 - RECORDS_OUT_OPERATOR_RS_8: 57 - RECORDS_OUT_OPERATOR_SEL_10: 57 - RECORDS_OUT_OPERATOR_SEL_13: 208 - RECORDS_OUT_OPERATOR_SEL_17: 208 - RECORDS_OUT_OPERATOR_SEL_2: 84 - RECORDS_OUT_OPERATOR_SEL_21: 127 - RECORDS_OUT_OPERATOR_SEL_6: 84 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_33: 84 + RECORDS_OUT_OPERATOR_RS_34: 208 + RECORDS_OUT_OPERATOR_SEL_31: 1 + RECORDS_OUT_OPERATOR_SEL_32: 1 + RECORDS_OUT_OPERATOR_SEL_35: 57 + RECORDS_OUT_OPERATOR_SEL_37: 127 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 292 +Stage-2 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -265,36 +257,33 @@ PREHOOK: Output: default@src2_n5 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 121 - CREATED_FILES: 125 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_IN_Map_4: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n5: 189 - RECORDS_OUT_INTERMEDIATE_Map_1: 57 - RECORDS_OUT_INTERMEDIATE_Map_4: 64 + RECORDS_OUT_INTERMEDIATE_Map_1: 84 + RECORDS_OUT_INTERMEDIATE_Map_4: 105 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_20: 84 - RECORDS_OUT_OPERATOR_FIL_29: 105 - RECORDS_OUT_OPERATOR_FS_16: 121 - RECORDS_OUT_OPERATOR_FS_24: 84 - RECORDS_OUT_OPERATOR_FS_33: 105 - RECORDS_OUT_OPERATOR_GBY_14: 121 - RECORDS_OUT_OPERATOR_GBY_26: 57 - RECORDS_OUT_OPERATOR_GBY_35: 64 + RECORDS_OUT_OPERATOR_FIL_33: 1 + RECORDS_OUT_OPERATOR_FIL_39: 1 + RECORDS_OUT_OPERATOR_FS_38: 189 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_27: 57 - RECORDS_OUT_OPERATOR_RS_36: 64 - RECORDS_OUT_OPERATOR_SEL_15: 121 - RECORDS_OUT_OPERATOR_SEL_21: 84 - RECORDS_OUT_OPERATOR_SEL_23: 84 - RECORDS_OUT_OPERATOR_SEL_25: 84 - RECORDS_OUT_OPERATOR_SEL_30: 105 - RECORDS_OUT_OPERATOR_SEL_32: 105 - RECORDS_OUT_OPERATOR_SEL_34: 105 - RECORDS_OUT_OPERATOR_TS_19: 500 - RECORDS_OUT_OPERATOR_TS_28: 500 + RECORDS_OUT_OPERATOR_RS_36: 84 + RECORDS_OUT_OPERATOR_RS_42: 105 + RECORDS_OUT_OPERATOR_SEL_34: 1 + RECORDS_OUT_OPERATOR_SEL_35: 1 + RECORDS_OUT_OPERATOR_SEL_37: 121 + RECORDS_OUT_OPERATOR_SEL_40: 1 + RECORDS_OUT_OPERATOR_SEL_41: 1 + RECORDS_OUT_OPERATOR_TS_21: 1 + RECORDS_OUT_OPERATOR_TS_27: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 11624 + NUM_DECODED_BATCHES: 2 + NUM_VECTOR_BATCHES: 2 + ROWS_EMITTED: 1000 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 @@ -315,36 +304,33 @@ PREHOOK: Output: default@src2_n5 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 63 - CREATED_FILES: 188 + CREATED_FILES: 184 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_IN_Map_4: 500 - RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2_n5: 292 - RECORDS_OUT_INTERMEDIATE_Map_1: 57 - RECORDS_OUT_INTERMEDIATE_Map_4: 127 + RECORDS_OUT_INTERMEDIATE_Map_1: 84 + RECORDS_OUT_INTERMEDIATE_Map_4: 208 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_20: 84 - RECORDS_OUT_OPERATOR_FIL_29: 208 - RECORDS_OUT_OPERATOR_FS_16: 184 - RECORDS_OUT_OPERATOR_FS_24: 84 - RECORDS_OUT_OPERATOR_FS_33: 208 - RECORDS_OUT_OPERATOR_GBY_14: 184 - RECORDS_OUT_OPERATOR_GBY_26: 57 - RECORDS_OUT_OPERATOR_GBY_35: 127 + RECORDS_OUT_OPERATOR_FIL_33: 1 + RECORDS_OUT_OPERATOR_FIL_39: 1 + RECORDS_OUT_OPERATOR_FS_38: 292 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_27: 57 - RECORDS_OUT_OPERATOR_RS_36: 127 - RECORDS_OUT_OPERATOR_SEL_15: 184 - RECORDS_OUT_OPERATOR_SEL_21: 84 - RECORDS_OUT_OPERATOR_SEL_23: 84 - RECORDS_OUT_OPERATOR_SEL_25: 84 - RECORDS_OUT_OPERATOR_SEL_30: 208 - RECORDS_OUT_OPERATOR_SEL_32: 208 - RECORDS_OUT_OPERATOR_SEL_34: 208 - RECORDS_OUT_OPERATOR_TS_19: 500 - RECORDS_OUT_OPERATOR_TS_28: 500 + RECORDS_OUT_OPERATOR_RS_36: 84 + RECORDS_OUT_OPERATOR_RS_42: 208 + RECORDS_OUT_OPERATOR_SEL_34: 1 + RECORDS_OUT_OPERATOR_SEL_35: 1 + RECORDS_OUT_OPERATOR_SEL_37: 184 + RECORDS_OUT_OPERATOR_SEL_40: 1 + RECORDS_OUT_OPERATOR_SEL_41: 1 + RECORDS_OUT_OPERATOR_TS_21: 1 + RECORDS_OUT_OPERATOR_TS_27: 1 TOTAL_TABLE_ROWS_WRITTEN: 292 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 11624 + NUM_DECODED_BATCHES: 2 + NUM_VECTOR_BATCHES: 2 + ROWS_EMITTED: 1000 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 diff --git a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out b/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out index 101b343506..4e244bc6f4 100644 --- a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out @@ -17,25 +17,25 @@ PREHOOK: Output: default@src2_n3 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 57 - CREATED_FILES: 61 + CREATED_FILES: 57 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 57 RECORDS_OUT_1_default.src2_n3: 84 - RECORDS_OUT_INTERMEDIATE_Map_1: 57 + RECORDS_OUT_INTERMEDIATE_Map_1: 84 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 84 - RECORDS_OUT_OPERATOR_FS_11: 57 - RECORDS_OUT_OPERATOR_FS_4: 84 - RECORDS_OUT_OPERATOR_GBY_7: 57 - RECORDS_OUT_OPERATOR_GBY_9: 57 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 84 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 57 - RECORDS_OUT_OPERATOR_SEL_10: 57 - RECORDS_OUT_OPERATOR_SEL_2: 84 - RECORDS_OUT_OPERATOR_SEL_6: 84 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 84 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 57 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 84 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -48,25 +48,25 @@ PREHOOK: Output: default@src2_n3 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 64 - CREATED_FILES: 125 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n3: 189 - RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Map_1: 189 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 189 - RECORDS_OUT_OPERATOR_FS_11: 121 - RECORDS_OUT_OPERATOR_FS_4: 189 - RECORDS_OUT_OPERATOR_GBY_7: 121 - RECORDS_OUT_OPERATOR_GBY_9: 121 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 189 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 121 - RECORDS_OUT_OPERATOR_SEL_10: 121 - RECORDS_OUT_OPERATOR_SEL_2: 189 - RECORDS_OUT_OPERATOR_SEL_6: 189 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 189 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 121 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -87,25 +87,25 @@ PREHOOK: Output: default@src2_n3 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 121 - CREATED_FILES: 125 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n3: 189 - RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Map_1: 189 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 189 - RECORDS_OUT_OPERATOR_FS_11: 121 - RECORDS_OUT_OPERATOR_FS_4: 189 - RECORDS_OUT_OPERATOR_GBY_7: 121 - RECORDS_OUT_OPERATOR_GBY_9: 121 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 189 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 121 - RECORDS_OUT_OPERATOR_SEL_10: 121 - RECORDS_OUT_OPERATOR_SEL_2: 189 - RECORDS_OUT_OPERATOR_SEL_6: 189 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 189 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 121 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -118,25 +118,25 @@ PREHOOK: Output: default@src2_n3 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 63 - CREATED_FILES: 188 + CREATED_FILES: 184 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2_n3: 292 - RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Map_1: 292 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FIL_12: 292 - RECORDS_OUT_OPERATOR_FS_11: 184 - RECORDS_OUT_OPERATOR_FS_4: 292 - RECORDS_OUT_OPERATOR_GBY_7: 184 - RECORDS_OUT_OPERATOR_GBY_9: 184 + RECORDS_OUT_OPERATOR_FIL_15: 1 + RECORDS_OUT_OPERATOR_FS_19: 292 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_8: 184 - RECORDS_OUT_OPERATOR_SEL_10: 184 - RECORDS_OUT_OPERATOR_SEL_2: 292 - RECORDS_OUT_OPERATOR_SEL_6: 292 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_17: 292 + RECORDS_OUT_OPERATOR_SEL_16: 1 + RECORDS_OUT_OPERATOR_SEL_18: 184 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 292 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -166,36 +166,32 @@ PREHOOK: Output: default@src3_n0 Stage-2 FILE SYSTEM COUNTERS: Stage-2 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 121 - CREATED_FILES: 129 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n3: 84 RECORDS_OUT_2_default.src3_n0: 105 - RECORDS_OUT_INTERMEDIATE_Map_1: 121 + RECORDS_OUT_INTERMEDIATE_Map_1: 189 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_23: 84 - RECORDS_OUT_OPERATOR_FIL_24: 105 - RECORDS_OUT_OPERATOR_FS_11: 57 - RECORDS_OUT_OPERATOR_FS_15: 105 - RECORDS_OUT_OPERATOR_FS_22: 64 - RECORDS_OUT_OPERATOR_FS_4: 84 - RECORDS_OUT_OPERATOR_GBY_18: 64 - RECORDS_OUT_OPERATOR_GBY_20: 64 - RECORDS_OUT_OPERATOR_GBY_7: 57 - RECORDS_OUT_OPERATOR_GBY_9: 57 + RECORDS_OUT_OPERATOR_FIL_29: 1 + RECORDS_OUT_OPERATOR_FIL_30: 1 + RECORDS_OUT_OPERATOR_FS_36: 84 + RECORDS_OUT_OPERATOR_FS_38: 105 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_19: 64 - RECORDS_OUT_OPERATOR_RS_8: 57 - RECORDS_OUT_OPERATOR_SEL_10: 57 - RECORDS_OUT_OPERATOR_SEL_13: 105 - RECORDS_OUT_OPERATOR_SEL_17: 105 - RECORDS_OUT_OPERATOR_SEL_2: 84 - RECORDS_OUT_OPERATOR_SEL_21: 64 - RECORDS_OUT_OPERATOR_SEL_6: 84 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_33: 84 + RECORDS_OUT_OPERATOR_RS_34: 105 + RECORDS_OUT_OPERATOR_SEL_31: 1 + RECORDS_OUT_OPERATOR_SEL_32: 1 + RECORDS_OUT_OPERATOR_SEL_35: 57 + RECORDS_OUT_OPERATOR_SEL_37: 64 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-2 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -211,36 +207,32 @@ PREHOOK: Output: default@src3_n0 Stage-2 FILE SYSTEM COUNTERS: Stage-2 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 63 - CREATED_FILES: 192 + CREATED_FILES: 184 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 - RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2_n3: 84 RECORDS_OUT_2_default.src3_n0: 208 - RECORDS_OUT_INTERMEDIATE_Map_1: 184 + RECORDS_OUT_INTERMEDIATE_Map_1: 292 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_23: 84 - RECORDS_OUT_OPERATOR_FIL_24: 208 - RECORDS_OUT_OPERATOR_FS_11: 57 - RECORDS_OUT_OPERATOR_FS_15: 208 - RECORDS_OUT_OPERATOR_FS_22: 127 - RECORDS_OUT_OPERATOR_FS_4: 84 - RECORDS_OUT_OPERATOR_GBY_18: 127 - RECORDS_OUT_OPERATOR_GBY_20: 127 - RECORDS_OUT_OPERATOR_GBY_7: 57 - RECORDS_OUT_OPERATOR_GBY_9: 57 + RECORDS_OUT_OPERATOR_FIL_29: 1 + RECORDS_OUT_OPERATOR_FIL_30: 1 + RECORDS_OUT_OPERATOR_FS_36: 84 + RECORDS_OUT_OPERATOR_FS_38: 208 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_19: 127 - RECORDS_OUT_OPERATOR_RS_8: 57 - RECORDS_OUT_OPERATOR_SEL_10: 57 - RECORDS_OUT_OPERATOR_SEL_13: 208 - RECORDS_OUT_OPERATOR_SEL_17: 208 - RECORDS_OUT_OPERATOR_SEL_2: 84 - RECORDS_OUT_OPERATOR_SEL_21: 127 - RECORDS_OUT_OPERATOR_SEL_6: 84 - RECORDS_OUT_OPERATOR_TS_0: 500 + RECORDS_OUT_OPERATOR_RS_33: 84 + RECORDS_OUT_OPERATOR_RS_34: 208 + RECORDS_OUT_OPERATOR_SEL_31: 1 + RECORDS_OUT_OPERATOR_SEL_32: 1 + RECORDS_OUT_OPERATOR_SEL_35: 57 + RECORDS_OUT_OPERATOR_SEL_37: 127 + RECORDS_OUT_OPERATOR_TS_0: 1 TOTAL_TABLE_ROWS_WRITTEN: 292 +Stage-2 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 5812 + NUM_DECODED_BATCHES: 1 + NUM_VECTOR_BATCHES: 1 + ROWS_EMITTED: 500 Stage-2 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 INPUT_DIRECTORIES_Map_1: 1 @@ -265,36 +257,33 @@ PREHOOK: Output: default@src2_n3 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 121 - CREATED_FILES: 125 + CREATED_FILES: 121 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_IN_Map_4: 500 - RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2_n3: 189 - RECORDS_OUT_INTERMEDIATE_Map_1: 57 - RECORDS_OUT_INTERMEDIATE_Map_4: 64 + RECORDS_OUT_INTERMEDIATE_Map_1: 84 + RECORDS_OUT_INTERMEDIATE_Map_4: 105 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_20: 84 - RECORDS_OUT_OPERATOR_FIL_29: 105 - RECORDS_OUT_OPERATOR_FS_16: 121 - RECORDS_OUT_OPERATOR_FS_24: 84 - RECORDS_OUT_OPERATOR_FS_33: 105 - RECORDS_OUT_OPERATOR_GBY_14: 121 - RECORDS_OUT_OPERATOR_GBY_26: 57 - RECORDS_OUT_OPERATOR_GBY_35: 64 + RECORDS_OUT_OPERATOR_FIL_33: 1 + RECORDS_OUT_OPERATOR_FIL_39: 1 + RECORDS_OUT_OPERATOR_FS_38: 189 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_27: 57 - RECORDS_OUT_OPERATOR_RS_36: 64 - RECORDS_OUT_OPERATOR_SEL_15: 121 - RECORDS_OUT_OPERATOR_SEL_21: 84 - RECORDS_OUT_OPERATOR_SEL_23: 84 - RECORDS_OUT_OPERATOR_SEL_25: 84 - RECORDS_OUT_OPERATOR_SEL_30: 105 - RECORDS_OUT_OPERATOR_SEL_32: 105 - RECORDS_OUT_OPERATOR_SEL_34: 105 - RECORDS_OUT_OPERATOR_TS_19: 500 - RECORDS_OUT_OPERATOR_TS_28: 500 + RECORDS_OUT_OPERATOR_RS_36: 84 + RECORDS_OUT_OPERATOR_RS_42: 105 + RECORDS_OUT_OPERATOR_SEL_34: 1 + RECORDS_OUT_OPERATOR_SEL_35: 1 + RECORDS_OUT_OPERATOR_SEL_37: 121 + RECORDS_OUT_OPERATOR_SEL_40: 1 + RECORDS_OUT_OPERATOR_SEL_41: 1 + RECORDS_OUT_OPERATOR_TS_21: 1 + RECORDS_OUT_OPERATOR_TS_27: 1 TOTAL_TABLE_ROWS_WRITTEN: 189 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 11624 + NUM_DECODED_BATCHES: 2 + NUM_VECTOR_BATCHES: 2 + ROWS_EMITTED: 1000 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 @@ -315,36 +304,33 @@ PREHOOK: Output: default@src2_n3 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 63 - CREATED_FILES: 188 + CREATED_FILES: 184 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_IN_Map_4: 500 - RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2_n3: 292 - RECORDS_OUT_INTERMEDIATE_Map_1: 57 - RECORDS_OUT_INTERMEDIATE_Map_4: 127 + RECORDS_OUT_INTERMEDIATE_Map_1: 84 + RECORDS_OUT_INTERMEDIATE_Map_4: 208 RECORDS_OUT_INTERMEDIATE_Reducer_3: 0 - RECORDS_OUT_OPERATOR_FIL_20: 84 - RECORDS_OUT_OPERATOR_FIL_29: 208 - RECORDS_OUT_OPERATOR_FS_16: 184 - RECORDS_OUT_OPERATOR_FS_24: 84 - RECORDS_OUT_OPERATOR_FS_33: 208 - RECORDS_OUT_OPERATOR_GBY_14: 184 - RECORDS_OUT_OPERATOR_GBY_26: 57 - RECORDS_OUT_OPERATOR_GBY_35: 127 + RECORDS_OUT_OPERATOR_FIL_33: 1 + RECORDS_OUT_OPERATOR_FIL_39: 1 + RECORDS_OUT_OPERATOR_FS_38: 292 RECORDS_OUT_OPERATOR_MAP_0: 0 - RECORDS_OUT_OPERATOR_RS_27: 57 - RECORDS_OUT_OPERATOR_RS_36: 127 - RECORDS_OUT_OPERATOR_SEL_15: 184 - RECORDS_OUT_OPERATOR_SEL_21: 84 - RECORDS_OUT_OPERATOR_SEL_23: 84 - RECORDS_OUT_OPERATOR_SEL_25: 84 - RECORDS_OUT_OPERATOR_SEL_30: 208 - RECORDS_OUT_OPERATOR_SEL_32: 208 - RECORDS_OUT_OPERATOR_SEL_34: 208 - RECORDS_OUT_OPERATOR_TS_19: 500 - RECORDS_OUT_OPERATOR_TS_28: 500 + RECORDS_OUT_OPERATOR_RS_36: 84 + RECORDS_OUT_OPERATOR_RS_42: 208 + RECORDS_OUT_OPERATOR_SEL_34: 1 + RECORDS_OUT_OPERATOR_SEL_35: 1 + RECORDS_OUT_OPERATOR_SEL_37: 184 + RECORDS_OUT_OPERATOR_SEL_40: 1 + RECORDS_OUT_OPERATOR_SEL_41: 1 + RECORDS_OUT_OPERATOR_TS_21: 1 + RECORDS_OUT_OPERATOR_TS_27: 1 TOTAL_TABLE_ROWS_WRITTEN: 292 +Stage-1 LLAP IO COUNTERS: + CACHE_HIT_BYTES: 11624 + NUM_DECODED_BATCHES: 2 + NUM_VECTOR_BATCHES: 2 + ROWS_EMITTED: 1000 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 1 GROUPED_INPUT_SPLITS_Map_4: 1 diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index 62fdbdba42..b20593848a 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -189,7 +189,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col0 (type: smallint) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -199,7 +198,6 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -275,7 +273,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -293,7 +290,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap @@ -301,7 +297,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -374,7 +369,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -384,11 +378,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -456,7 +449,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -466,11 +458,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -606,7 +597,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col0 (type: smallint) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -616,7 +606,6 @@ STAGE PLANS: Select Operator expressions: KEY._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -692,7 +681,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -710,7 +698,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap @@ -718,7 +705,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -791,7 +777,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -801,11 +786,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -873,7 +857,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -883,11 +866,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1171,7 +1153,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4648 + totalSize 4670 #### A masked pattern was here #### # Storage Information @@ -1249,7 +1231,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4658 + totalSize 4656 #### A masked pattern was here #### # Storage Information @@ -1390,7 +1372,6 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1407,63 +1388,27 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + key expressions: _col4 (type: tinyint), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: tinyint) + expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.over1k_part2_orc - Select Operator - expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) - outputColumnNames: si, i, b, f, ds, t - Statistics: Num rows: 11 Data size: 1221 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') - keys: ds (type: string), t (type: tinyint) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 5 Data size: 8935 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: tinyint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) - Statistics: Num rows: 5 Data size: 8935 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 5 Data size: 9255 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 5 Data size: 9255 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 9255 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -1527,7 +1472,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -1537,7 +1481,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1607,7 +1550,6 @@ STAGE PLANS: key expressions: _col2 (type: int) sort order: + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs @@ -1632,7 +1574,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: vectorized, llap @@ -1640,7 +1581,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1694,7 +1634,6 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1714,7 +1653,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) sort order: +++++ - Map-reduce partition columns: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: vectorized, llap LLAP IO: all inputs @@ -1730,50 +1669,21 @@ STAGE PLANS: expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.over1k_part2_orc Select Operator - expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) - outputColumnNames: si, i, b, f, ds, t - Statistics: Num rows: 5 Data size: 555 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') - keys: ds (type: string), t (type: tinyint) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 3574 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: tinyint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) - Statistics: Num rows: 2 Data size: 3574 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 3702 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 2 Data size: 3702 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 3702 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2_orc Stage: Stage-2 Dependency Collection @@ -1842,7 +1752,7 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: vectorized, llap + Execution mode: llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: tinyint), KEY._col1 (type: smallint), KEY._col2 (type: int), KEY._col3 (type: bigint), KEY._col4 (type: float) @@ -1853,15 +1763,21 @@ STAGE PLANS: expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.over1k_part2_orc + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 5 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2_orc Stage: Stage-2 Dependency Collection @@ -2210,7 +2126,6 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2227,64 +2142,27 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - key expressions: _col3 (type: float) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number File Output Operator compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.over1k_part_buck_sort2_orc - Select Operator - expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) - outputColumnNames: si, i, b, f, t - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') - keys: t (type: tinyint) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 5 Data size: 8500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 5 Data size: 8500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: tinyint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 5 Data size: 8820 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 5 Data size: 8820 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 5 Data size: 8820 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -2347,7 +2225,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: vectorized, llap LLAP IO: all inputs @@ -2357,11 +2234,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 11 Data size: 2288 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2906,7 +2782,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: int), _col1 (type: smallint), _col2 (type: string) Execution mode: vectorized, llap LLAP IO: all inputs @@ -2924,7 +2799,6 @@ STAGE PLANS: key expressions: _col2 (type: string), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col2 (type: string) - Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: smallint) Reducer 3 Execution mode: vectorized, llap @@ -2932,11 +2806,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: smallint), KEY._col2 (type: string), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _bucket_number - Statistics: Num rows: 10 Data size: 2656 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 10 Data size: 2656 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 816 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out index ec95389d08..4101783687 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization.q.out @@ -146,7 +146,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -156,7 +155,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -232,7 +230,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs @@ -250,7 +247,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: llap @@ -258,7 +254,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -331,7 +326,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -341,7 +335,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -413,7 +406,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -423,7 +415,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -563,7 +554,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -573,7 +563,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -649,7 +638,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Execution mode: llap LLAP IO: no inputs @@ -667,7 +655,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: llap @@ -675,7 +662,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -748,7 +734,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -758,7 +743,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -830,7 +814,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -840,7 +823,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -1347,7 +1329,6 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1364,63 +1345,27 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col1 (type: int) - sort order: + - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + key expressions: _col4 (type: tinyint), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), KEY.reducesinkkey0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: tinyint) + expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.over1k_part2 - Select Operator - expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) - outputColumnNames: si, i, b, f, ds, t - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') - keys: ds (type: string), t (type: tinyint) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: tinyint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -1484,7 +1429,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _col1 (type: int) sort order: ++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col2 (type: bigint), _col3 (type: float) Execution mode: llap LLAP IO: no inputs @@ -1494,7 +1438,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), KEY._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1564,7 +1507,6 @@ STAGE PLANS: key expressions: _col2 (type: int) sort order: + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float) Execution mode: llap LLAP IO: no inputs @@ -1589,7 +1531,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint) sort order: + Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) Reducer 3 Execution mode: llap @@ -1597,7 +1538,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -1651,7 +1591,6 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1671,7 +1610,7 @@ STAGE PLANS: Reduce Output Operator key expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) sort order: +++++ - Map-reduce partition columns: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float) + Map-reduce partition columns: _col0 (type: tinyint) Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs @@ -1687,50 +1626,21 @@ STAGE PLANS: expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.over1k_part2 Select Operator - expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) - outputColumnNames: si, i, b, f, ds, t - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') - keys: ds (type: string), t (type: tinyint) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: tinyint) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 Stage: Stage-2 Dependency Collection @@ -1810,15 +1720,21 @@ STAGE PLANS: expressions: _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col0 (type: tinyint) outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.over1k_part2 + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part2 Stage: Stage-2 Dependency Collection @@ -2167,7 +2083,6 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2184,64 +2099,27 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col3 (type: float) - sort order: + - Map-reduce partition columns: _col0 (type: smallint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) + key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number File Output Operator compressed: false + Dp Sort State: PARTITION_BUCKET_SORTED Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.over1k_part_buck_sort2 - Select Operator - expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) - outputColumnNames: si, i, b, f, t - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') - keys: t (type: tinyint) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: tinyint) - sort order: + - Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct) - Reducer 3 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) - keys: KEY._col0 (type: tinyint) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col0 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -2304,7 +2182,6 @@ STAGE PLANS: key expressions: _col4 (type: tinyint), _bucket_number (type: string), _col3 (type: float) sort order: +++ Map-reduce partition columns: _col4 (type: tinyint) - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint) Execution mode: llap LLAP IO: no inputs @@ -2314,7 +2191,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY._col3 (type: float), KEY._col4 (type: tinyint), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED @@ -2680,7 +2556,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2690,7 +2565,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2764,7 +2638,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2774,7 +2647,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2848,7 +2720,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2858,7 +2729,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -2932,7 +2802,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -2942,7 +2811,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -3016,7 +2884,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -3026,7 +2893,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -3100,7 +2966,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), _col5 (type: int) - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float) Execution mode: llap LLAP IO: no inputs @@ -3110,7 +2975,6 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 1 Data size: 208 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -3584,3 +3448,471 @@ POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680 POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599 #### A masked pattern was here #### 17814641134 +PREHOOK: query: drop table over1k_n3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: drop table over1k_n3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: create table over1k_n3( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal(4,2), + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over1k_n3 +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k_n3 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over1k_n3 +PREHOOK: query: analyze table over1k_n3 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +POSTHOOK: query: analyze table over1k_n3 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@over1k_n3 +POSTHOOK: Output: default@over1k_n3 +#### A masked pattern was here #### +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 10 Data size: 1110 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5 Data size: 8935 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 5 Data size: 8935 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5 Data size: 9255 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5 Data size: 9255 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 5 Data size: 9255 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t>27 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t > 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t > 27Y) or t is null) (type: boolean) + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 352 Data size: 8448 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + Select Operator + expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), 'foo' (type: string), _col4 (type: tinyint) + outputColumnNames: si, i, b, f, ds, t + Statistics: Num rows: 352 Data size: 39072 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: compute_stats(si, 'hll'), compute_stats(i, 'hll'), compute_stats(b, 'hll'), compute_stats(f, 'hll') + keys: ds (type: string), t (type: tinyint) + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: tinyint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: tinyint) + Statistics: Num rows: 129 Data size: 230523 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1), compute_stats(VALUE._col2), compute_stats(VALUE._col3) + keys: KEY._col0 (type: string), KEY._col1 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col2 (type: struct), _col3 (type: struct), _col4 (type: struct), _col5 (type: struct), _col0 (type: string), _col1 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 128 Data size: 236928 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + +PREHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k_n3 +PREHOOK: Output: default@over1k_part@ds=foo +POSTHOOK: query: explain insert overwrite table over1k_part partition(ds="foo", t) select si,i,b,f,t from over1k_n3 where t is null or t=27 limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k_n3 +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over1k_n3 + filterExpr: (t is null or (t = 27Y)) (type: boolean) + Statistics: Num rows: 1049 Data size: 25160 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: ((t = 27Y) or t is null) (type: boolean) + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: si (type: smallint), i (type: int), b (type: bigint), f (type: float), t (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 11 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col4 (type: tinyint) + sort order: + + Map-reduce partition columns: _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), KEY._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + partition: + ds foo + t + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.over1k_part + + Stage: Stage-3 + Stats Work + Basic Stats Work: + Column Stats Desc: + Columns: si, i, b, f + Column Types: smallint, int, bigint, float + Table: default.over1k_part + diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index d953eebefd..ece03404ce 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -352,19 +352,17 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) - sort order: + + sort order: ++ Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) + value expressions: _col0 (type: float), _col1 (type: float) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), VALUE._col2 (type: int) + expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), KEY._col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false Dp Sort State: PARTITION_SORTED @@ -602,7 +600,7 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: float), KEY._col2 (type: float) @@ -615,33 +613,13 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.ss_part - Select Operator - expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) - outputColumnNames: ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(ss_net_paid_inc_tax, 'hll'), compute_stats(ss_net_profit, 'hll') - keys: ss_sold_date_sk (type: int) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -863,48 +841,26 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: int) - sort order: + + sort order: ++ Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) + value expressions: _col0 (type: float), _col1 (type: float) Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), VALUE._col2 (type: int) + expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), KEY._col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.ss_part - Select Operator - expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) - outputColumnNames: ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: compute_stats(ss_net_paid_inc_tax, 'hll'), compute_stats(ss_net_profit, 'hll') - keys: ss_sold_date_sk (type: int) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -1188,7 +1144,7 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: float), KEY._col2 (type: float) @@ -1201,33 +1157,13 @@ STAGE PLANS: Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.ss_part_orc - Select Operator - expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) - outputColumnNames: ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(ss_net_paid_inc_tax, 'hll'), compute_stats(ss_net_profit, 'hll') - keys: ss_sold_date_sk (type: int) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection @@ -1449,48 +1385,26 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col2 (type: int) - sort order: + + sort order: ++ Map-reduce partition columns: _col2 (type: int) - Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) + value expressions: _col0 (type: float), _col1 (type: float) Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), VALUE._col2 (type: int) + expressions: VALUE._col0 (type: float), VALUE._col1 (type: float), KEY._col2 (type: int) outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false + Dp Sort State: PARTITION_SORTED Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde name: default.ss_part_orc - Select Operator - expressions: _col0 (type: float), _col1 (type: float), _col2 (type: int) - outputColumnNames: ss_net_paid_inc_tax, ss_net_profit, ss_sold_date_sk - Statistics: Num rows: 8 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(ss_net_paid_inc_tax, 'hll'), compute_stats(ss_net_profit, 'hll') - keys: ss_sold_date_sk (type: int) - mode: complete - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col2 (type: struct), _col0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 5fd1bf6fb2..dfe7bd6b34 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -512,23 +512,23 @@ STAGE PLANS: outputColumnNames: _col0, _col3 Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col3 (type: string), _bucket_number (type: string), _col0 (type: struct) - sort order: +++ - Map-reduce partition columns: _col3 (type: string) + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col3 (type: string) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), 'foo' (type: string), 'bar' (type: string), KEY._col3 (type: string), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number - Statistics: Num rows: 5 Data size: 2220 Basic stats: COMPLETE Column stats: PARTIAL + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2220 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2170 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -962,7 +962,7 @@ STAGE PLANS: TableScan alias: acid_2l_part filterExpr: (value = 'bar') (type: boolean) - Statistics: Num rows: 4200 Data size: 1253037 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4200 Data size: 1243317 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (value = 'bar') (type: boolean) Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL @@ -1283,23 +1283,23 @@ STAGE PLANS: outputColumnNames: _col0, _col4 Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: '2008-04-08' (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) - sort order: ++++ - Map-reduce partition columns: '2008-04-08' (type: string), _col4 (type: int) + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL + expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), VALUE._col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 1790 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1740 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1419,7 +1419,7 @@ STAGE PLANS: TableScan alias: acid_2l_part_sdpo filterExpr: (value = 'bar') (type: boolean) - Statistics: Num rows: 4200 Data size: 1253037 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 4200 Data size: 1243317 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (value = 'bar') (type: boolean) Statistics: Num rows: 5 Data size: 1375 Basic stats: COMPLETE Column stats: PARTIAL @@ -1428,23 +1428,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: int), _bucket_number (type: string), _col0 (type: struct) - sort order: ++++ - Map-reduce partition columns: _col1 (type: string), _col2 (type: int) + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL + value expressions: _col1 (type: string), _col2 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), KEY._col1 (type: string), KEY._col2 (type: int), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _bucket_number - Statistics: Num rows: 5 Data size: 2240 Basic stats: COMPLETE Column stats: PARTIAL + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), VALUE._col1 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 2240 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 1320 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1633,24 +1633,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) - sort order: ++++ - Map-reduce partition columns: _col3 (type: string), _col4 (type: int) + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), 'bar' (type: string) + value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1742,24 +1741,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col3, _col4 Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator - key expressions: _col3 (type: string), _col4 (type: int), _bucket_number (type: string), _col0 (type: struct) - sort order: ++++ - Map-reduce partition columns: _col3 (type: string), _col4 (type: int) + key expressions: _col0 (type: struct) + sort order: + + Map-reduce partition columns: UDFToInteger(_col0) (type: int) Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL - value expressions: _col1 (type: string), 'bar' (type: string) + value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: int) Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 Execution mode: llap Reduce Operator Tree: Select Operator - expressions: KEY._col0 (type: struct), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._bucket_number (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL + expressions: KEY.reducesinkkey0 (type: struct), VALUE._col0 (type: string), 'bar' (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 5 Data size: 3595 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 5 Data size: 2675 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/llap_acid.q.out b/ql/src/test/results/clientpositive/llap/llap_acid.q.out index b8a4058f6c..feb4006514 100644 --- a/ql/src/test/results/clientpositive/llap/llap_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_acid.q.out @@ -97,82 +97,8 @@ POSTHOOK: Input: default@orc_llap_n1@csmallint=2 POSTHOOK: Input: default@orc_llap_n1@csmallint=3 #### A masked pattern was here #### PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Vertices: - Map 1 - Map Operator Tree: - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cint:int, 1:cbigint:bigint, 2:cfloat:float, 3:cdouble:double, 4:csmallint:smallint, 5:ROW__ID:struct] - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 4, 1] - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 4:smallint, 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 1:bigint - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: true - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - includeColumns: [0, 1] - dataColumns: cint:int, cbigint:bigint, cfloat:float, cdouble:double - partitionColumnCount: 1 - partitionColumns: csmallint:smallint - scratchColumnTypeNames: [] - Reducer 2 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY.reducesinkkey0:smallint, KEY.reducesinkkey1:int, VALUE._col0:bigint - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [1, 0, 2] - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - - Stage: Stage-0 - Fetch Operator + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] PREHOOK: query: select cint, csmallint, cbigint from orc_llap_n1 where cint is not null order by csmallint, cint @@ -253,90 +179,8 @@ POSTHOOK: Output: default@orc_llap_n1@csmallint=1 POSTHOOK: Output: default@orc_llap_n1@csmallint=2 POSTHOOK: Output: default@orc_llap_n1@csmallint=3 PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Tez - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Vertices: - Map 1 - Map Operator Tree: - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cint:int, 1:cbigint:bigint, 2:cfloat:float, 3:cdouble:double, 4:csmallint:smallint, 5:ROW__ID:struct] - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColEqualLongScalar(col 0:int, val 1) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [5, 2, 3, 4] - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 5:struct - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumns: 6:int - valueColumns: 2:float, 3:double, 4:smallint - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: true - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - includeColumns: [0, 2, 3] - dataColumns: cint:int, cbigint:bigint, cfloat:float, cdouble:double - neededVirtualColumns: [ROWID] - partitionColumnCount: 1 - partitionColumns: csmallint:smallint - scratchColumnTypeNames: [bigint] - Reducer 2 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: z - reduceColumnSortOrder: + - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - dataColumns: KEY.reducesinkkey0:struct, VALUE._col1:float, VALUE._col2:double, VALUE._col3:smallint - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, bigint] - Reduce Operator Tree: - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 4, 5, 1, 2, 3] - selectExpressions: ConstantVectorExpression(val 1) -> 4:int, ConstantVectorExpression(val 2) -> 5:bigint - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - - Stage: Stage-2 - - Stage: Stage-0 - - Stage: Stage-3 + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] PREHOOK: query: update orc_llap_n1 set cbigint = 2 where cint = 1 PREHOOK: type: QUERY @@ -375,82 +219,8 @@ POSTHOOK: Input: default@orc_llap_n1@csmallint=2 POSTHOOK: Input: default@orc_llap_n1@csmallint=3 #### A masked pattern was here #### PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Vertices: - Map 1 - Map Operator Tree: - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cint:int, 1:cbigint:bigint, 2:cfloat:float, 3:cdouble:double, 4:csmallint:smallint, 5:ROW__ID:struct] - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 4, 1] - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 4:smallint, 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 1:bigint - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: true - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - includeColumns: [0, 1] - dataColumns: cint:int, cbigint:bigint, cfloat:float, cdouble:double - partitionColumnCount: 1 - partitionColumns: csmallint:smallint - scratchColumnTypeNames: [] - Reducer 2 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY.reducesinkkey0:smallint, KEY.reducesinkkey1:int, VALUE._col0:bigint - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [1, 0, 2] - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - - Stage: Stage-0 - Fetch Operator + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] PREHOOK: query: select cint, csmallint, cbigint from orc_llap_n1 where cint is not null order by csmallint, cint diff --git a/ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out b/ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out index b91c3fffa7..a0981f2902 100644 --- a/ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_acid_fast.q.out @@ -91,82 +91,8 @@ POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 #### A masked pattern was here #### PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Vertices: - Map 1 - Map Operator Tree: - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cint:int, 1:cbigint:bigint, 2:cfloat:float, 3:cdouble:double, 4:csmallint:smallint, 5:ROW__ID:struct] - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 4, 1] - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 4:smallint, 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 1:bigint - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: true - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - includeColumns: [0, 1] - dataColumns: cint:int, cbigint:bigint, cfloat:float, cdouble:double - partitionColumnCount: 1 - partitionColumns: csmallint:smallint - scratchColumnTypeNames: [] - Reducer 2 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY.reducesinkkey0:smallint, KEY.reducesinkkey1:int, VALUE._col0:bigint - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [1, 0, 2] - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - - Stage: Stage-0 - Fetch Operator + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] PREHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order by csmallint, cint @@ -247,90 +173,8 @@ POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=1 POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=2 POSTHOOK: Output: default@orc_llap_acid_fast@csmallint=3 PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-1 - Tez - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Vertices: - Map 1 - Map Operator Tree: - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cint:int, 1:cbigint:bigint, 2:cfloat:float, 3:cdouble:double, 4:csmallint:smallint, 5:ROW__ID:struct] - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: FilterLongColEqualLongScalar(col 0:int, val 1) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [5, 2, 3, 4] - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 5:struct - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumns: 6:int - valueColumns: 2:float, 3:double, 4:smallint - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: true - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - includeColumns: [0, 2, 3] - dataColumns: cint:int, cbigint:bigint, cfloat:float, cdouble:double - neededVirtualColumns: [ROWID] - partitionColumnCount: 1 - partitionColumns: csmallint:smallint - scratchColumnTypeNames: [bigint] - Reducer 2 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: z - reduceColumnSortOrder: + - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - dataColumns: KEY.reducesinkkey0:struct, VALUE._col1:float, VALUE._col2:double, VALUE._col3:smallint - partitionColumnCount: 0 - scratchColumnTypeNames: [bigint, bigint] - Reduce Operator Tree: - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 4, 5, 1, 2, 3] - selectExpressions: ConstantVectorExpression(val 1) -> 4:int, ConstantVectorExpression(val 2) -> 5:bigint - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - - Stage: Stage-2 - - Stage: Stage-0 - - Stage: Stage-3 + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] PREHOOK: query: update orc_llap_acid_fast set cbigint = 2 where cint = 1 PREHOOK: type: QUERY @@ -369,82 +213,8 @@ POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=2 POSTHOOK: Input: default@orc_llap_acid_fast@csmallint=3 #### A masked pattern was here #### PLAN VECTORIZATION: - enabled: true - enabledConditionsMet: [hive.vectorized.execution.enabled IS true] - -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Vertices: - Map 1 - Map Operator Tree: - TableScan Vectorization: - native: true - vectorizationSchemaColumns: [0:cint:int, 1:cbigint:bigint, 2:cfloat:float, 3:cdouble:double, 4:csmallint:smallint, 5:ROW__ID:struct] - Filter Vectorization: - className: VectorFilterOperator - native: true - predicateExpression: SelectColumnIsNotNull(col 0:int) - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 4, 1] - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumns: 4:smallint, 0:int - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumns: 1:bigint - Execution mode: vectorized, llap - LLAP IO: may be used (ACID table) - Map Vectorization: - enabled: true - enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true - inputFormatFeatureSupport: [DECIMAL_64] - featureSupportInUse: [DECIMAL_64] - inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - allNative: true - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 4 - includeColumns: [0, 1] - dataColumns: cint:int, cbigint:bigint, cfloat:float, cdouble:double - partitionColumnCount: 1 - partitionColumns: csmallint:smallint - scratchColumnTypeNames: [] - Reducer 2 - Execution mode: vectorized, llap - Reduce Vectorization: - enabled: true - enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: zz - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY.reducesinkkey0:smallint, KEY.reducesinkkey1:int, VALUE._col0:bigint - partitionColumnCount: 0 - scratchColumnTypeNames: [] - Reduce Operator Tree: - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [1, 0, 2] - File Sink Vectorization: - className: VectorFileSinkOperator - native: false - - Stage: Stage-0 - Fetch Operator + enabled: false + enabledConditionsNotMet: [hive.vectorized.execution.enabled IS false] PREHOOK: query: select cint, csmallint, cbigint from orc_llap_acid_fast where cint is not null order by csmallint, cint diff --git a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index 7e6e88bd08..18dca4921f 100644 --- a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -1916,7 +1916,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: oft - Statistics: Num rows: 12288 Data size: 1884148 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 13243096 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true vectorizationSchemaColumns: [0:csmallint:smallint, 1:cint:int, 2:cbigint:bigint, 3:cfloat:float, 4:cdouble:double, 5:cstring1:string, 6:cchar1:char(255), 7:cvchar1:varchar(255), 8:cboolean1:boolean, 9:cboolean2:boolean, 10:ctinyint:tinyint, 11:ROW__ID:struct] @@ -1939,7 +1939,7 @@ STAGE PLANS: outputColumnNames: _col1, _col6, _col7, _col10 input vertices: 1 Map 2 - Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: _col10 (type: tinyint), _col1 (type: int), _col6 (type: char(255)), _col7 (type: varchar(255)) outputColumnNames: _col0, _col1, _col2, _col3 @@ -1947,13 +1947,13 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [10, 1, 6, 7] - Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 960 Data size: 4040 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 960 Data size: 240494 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/llap_smb.q.out b/ql/src/test/results/clientpositive/llap/llap_smb.q.out index ed10999f8f..d2593b19c9 100644 --- a/ql/src/test/results/clientpositive/llap/llap_smb.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_smb.q.out @@ -269,10 +269,10 @@ STAGE PLANS: TableScan alias: a filterExpr: id is not null (type: boolean) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: id is not null (type: boolean) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Merge Join Operator condition map: Inner Join 0 to 1 @@ -280,18 +280,18 @@ STAGE PLANS: 0 id (type: bigint) 1 id (type: bigint) outputColumnNames: _col2, _col3 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() keys: _col2 (type: int), _col3 (type: smallint) mode: hash outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: int), _col1 (type: smallint) sort order: ++ Map-reduce partition columns: _col0 (type: int), _col1 (type: smallint) - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col2 (type: bigint) Execution mode: llap Reducer 2 @@ -302,10 +302,10 @@ STAGE PLANS: keys: KEY._col0 (type: int), KEY._col1 (type: smallint) mode: mergepartial outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: ###Masked### Data size: ###Masked### Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out index 406c6204e5..ca3c08ab64 100644 --- a/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out +++ b/ql/src/test/results/clientpositive/llap/load_data_using_job.q.out @@ -1191,9 +1191,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1205,29 +1202,16 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string) + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 68 Data size: 24552 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -1268,124 +1252,124 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08 #### A masked pattern was here #### -8 val_8 2008-04-08 -33 val_33 2008-04-08 -37 val_37 2008-04-08 +165 val_165 2008-04-08 +484 val_484 2008-04-08 +150 val_150 2008-04-08 +224 val_224 2008-04-08 +66 val_66 2008-04-08 +213 val_213 2008-04-08 +374 val_374 2008-04-08 +495 val_495 2008-04-08 37 val_37 2008-04-08 +327 val_327 2008-04-08 +15 val_15 2008-04-08 +338 val_338 2008-04-08 +459 val_459 2008-04-08 +466 val_466 2008-04-08 +396 val_396 2008-04-08 +309 val_309 2008-04-08 +367 val_367 2008-04-08 +0 val_0 2008-04-08 +455 val_455 2008-04-08 +316 val_316 2008-04-08 +345 val_345 2008-04-08 +129 val_129 2008-04-08 +378 val_378 2008-04-08 +4 val_4 2008-04-08 +356 val_356 2008-04-08 +169 val_169 2008-04-08 +125 val_125 2008-04-08 +437 val_437 2008-04-08 +286 val_286 2008-04-08 +187 val_187 2008-04-08 +176 val_176 2008-04-08 +459 val_459 2008-04-08 51 val_51 2008-04-08 -51 val_51 2008-04-08 -103 val_103 2008-04-08 103 val_103 2008-04-08 -125 val_125 2008-04-08 -125 val_125 2008-04-08 -165 val_165 2008-04-08 -165 val_165 2008-04-08 -194 val_194 2008-04-08 -260 val_260 2008-04-08 +239 val_239 2008-04-08 +213 val_213 2008-04-08 +176 val_176 2008-04-08 275 val_275 2008-04-08 -305 val_305 2008-04-08 -323 val_323 2008-04-08 -411 val_411 2008-04-08 -455 val_455 2008-04-08 -466 val_466 2008-04-08 -466 val_466 2008-04-08 +260 val_260 2008-04-08 +404 val_404 2008-04-08 +217 val_217 2008-04-08 +84 val_84 2008-04-08 466 val_466 2008-04-08 -484 val_484 2008-04-08 -0 val_0 2008-04-08 -0 val_0 2008-04-08 +8 val_8 2008-04-08 +411 val_411 2008-04-08 +172 val_172 2008-04-08 +129 val_129 2008-04-08 +158 val_158 2008-04-08 0 val_0 2008-04-08 -4 val_4 2008-04-08 -11 val_11 2008-04-08 -19 val_19 2008-04-08 26 val_26 2008-04-08 -26 val_26 2008-04-08 -44 val_44 2008-04-08 -77 val_77 2008-04-08 -118 val_118 2008-04-08 -118 val_118 2008-04-08 -158 val_158 2008-04-08 -217 val_217 2008-04-08 -217 val_217 2008-04-08 -316 val_316 2008-04-08 -316 val_316 2008-04-08 -316 val_316 2008-04-08 -327 val_327 2008-04-08 -327 val_327 2008-04-08 +165 val_165 2008-04-08 327 val_327 2008-04-08 -338 val_338 2008-04-08 -356 val_356 2008-04-08 -378 val_378 2008-04-08 -389 val_389 2008-04-08 -404 val_404 2008-04-08 +51 val_51 2008-04-08 404 val_404 2008-04-08 -419 val_419 2008-04-08 -444 val_444 2008-04-08 -462 val_462 2008-04-08 -462 val_462 2008-04-08 -15 val_15 2008-04-08 -15 val_15 2008-04-08 -80 val_80 2008-04-08 -84 val_84 2008-04-08 -84 val_84 2008-04-08 -95 val_95 2008-04-08 95 val_95 2008-04-08 -129 val_129 2008-04-08 -129 val_129 2008-04-08 -169 val_169 2008-04-08 -169 val_169 2008-04-08 -169 val_169 2008-04-08 -169 val_169 2008-04-08 -239 val_239 2008-04-08 -239 val_239 2008-04-08 282 val_282 2008-04-08 +187 val_187 2008-04-08 +316 val_316 2008-04-08 +169 val_169 2008-04-08 +77 val_77 2008-04-08 +0 val_0 2008-04-08 +118 val_118 2008-04-08 282 val_282 2008-04-08 -367 val_367 2008-04-08 -367 val_367 2008-04-08 -66 val_66 2008-04-08 -114 val_114 2008-04-08 -136 val_136 2008-04-08 -143 val_143 2008-04-08 -172 val_172 2008-04-08 -172 val_172 2008-04-08 -190 val_190 2008-04-08 +419 val_419 2008-04-08 +15 val_15 2008-04-08 +118 val_118 2008-04-08 +19 val_19 2008-04-08 +224 val_224 2008-04-08 +309 val_309 2008-04-08 +389 val_389 2008-04-08 +327 val_327 2008-04-08 242 val_242 2008-04-08 +392 val_392 2008-04-08 242 val_242 2008-04-08 -286 val_286 2008-04-08 -345 val_345 2008-04-08 -400 val_400 2008-04-08 -437 val_437 2008-04-08 -448 val_448 2008-04-08 -459 val_459 2008-04-08 -459 val_459 2008-04-08 -150 val_150 2008-04-08 -176 val_176 2008-04-08 -176 val_176 2008-04-08 -183 val_183 2008-04-08 -187 val_187 2008-04-08 -187 val_187 2008-04-08 -187 val_187 2008-04-08 -202 val_202 2008-04-08 -213 val_213 2008-04-08 -213 val_213 2008-04-08 -224 val_224 2008-04-08 -224 val_224 2008-04-08 +396 val_396 2008-04-08 +95 val_95 2008-04-08 +11 val_11 2008-04-08 +143 val_143 2008-04-08 228 val_228 2008-04-08 +33 val_33 2008-04-08 +103 val_103 2008-04-08 +367 val_367 2008-04-08 +239 val_239 2008-04-08 +480 val_480 2008-04-08 +202 val_202 2008-04-08 +316 val_316 2008-04-08 235 val_235 2008-04-08 +80 val_80 2008-04-08 +44 val_44 2008-04-08 +466 val_466 2008-04-08 257 val_257 2008-04-08 -309 val_309 2008-04-08 -309 val_309 2008-04-08 -341 val_341 2008-04-08 -374 val_374 2008-04-08 -392 val_392 2008-04-08 -396 val_396 2008-04-08 -396 val_396 2008-04-08 +190 val_190 2008-04-08 +114 val_114 2008-04-08 396 val_396 2008-04-08 -477 val_477 2008-04-08 -480 val_480 2008-04-08 -480 val_480 2008-04-08 +217 val_217 2008-04-08 +125 val_125 2008-04-08 +187 val_187 2008-04-08 480 val_480 2008-04-08 491 val_491 2008-04-08 -495 val_495 2008-04-08 +305 val_305 2008-04-08 +444 val_444 2008-04-08 +169 val_169 2008-04-08 +323 val_323 2008-04-08 +480 val_480 2008-04-08 +136 val_136 2008-04-08 +172 val_172 2008-04-08 +462 val_462 2008-04-08 +26 val_26 2008-04-08 +462 val_462 2008-04-08 +341 val_341 2008-04-08 +183 val_183 2008-04-08 +84 val_84 2008-04-08 +37 val_37 2008-04-08 +448 val_448 2008-04-08 +194 val_194 2008-04-08 +477 val_477 2008-04-08 +169 val_169 2008-04-08 +400 val_400 2008-04-08 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -1418,9 +1402,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1432,29 +1413,16 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -1501,124 +1469,124 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### -8 val_8 2008-04-08 0 -33 val_33 2008-04-08 0 -103 val_103 2008-04-08 0 -125 val_125 2008-04-08 0 -275 val_275 2008-04-08 0 -305 val_305 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 77 val_77 2008-04-08 0 118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 118 val_118 2008-04-08 0 -158 val_158 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 +224 val_224 2008-04-08 0 389 val_389 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -419 val_419 2008-04-08 0 -80 val_80 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 95 val_95 2008-04-08 0 -129 val_129 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -282 val_282 2008-04-08 0 -367 val_367 2008-04-08 0 143 val_143 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 190 val_190 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -400 val_400 2008-04-08 0 -437 val_437 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -187 val_187 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -309 val_309 2008-04-08 0 -341 val_341 2008-04-08 0 396 val_396 2008-04-08 0 -477 val_477 2008-04-08 0 -480 val_480 2008-04-08 0 +125 val_125 2008-04-08 0 480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 480 val_480 2008-04-08 0 -495 val_495 2008-04-08 0 -37 val_37 2008-04-08 1 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 37 val_37 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -103 val_103 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 125 val_125 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -194 val_194 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 260 val_260 2008-04-08 1 -323 val_323 2008-04-08 1 -411 val_411 2008-04-08 1 -455 val_455 2008-04-08 1 +217 val_217 2008-04-08 1 466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 0 val_0 2008-04-08 1 -11 val_11 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 19 val_19 2008-04-08 1 -44 val_44 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 +309 val_309 2008-04-08 1 327 val_327 2008-04-08 1 -356 val_356 2008-04-08 1 -378 val_378 2008-04-08 1 -444 val_444 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -95 val_95 2008-04-08 1 -129 val_129 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -239 val_239 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 239 val_239 2008-04-08 1 -282 val_282 2008-04-08 1 -367 val_367 2008-04-08 1 -66 val_66 2008-04-08 1 -114 val_114 2008-04-08 1 -136 val_136 2008-04-08 1 -286 val_286 2008-04-08 1 -345 val_345 2008-04-08 1 -459 val_459 2008-04-08 1 -150 val_150 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 -183 val_183 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 202 val_202 2008-04-08 1 -228 val_228 2008-04-08 1 235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 257 val_257 2008-04-08 1 -309 val_309 2008-04-08 1 -374 val_374 2008-04-08 1 -392 val_392 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -1651,9 +1619,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1665,29 +1630,16 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 147 Data size: 53016 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -1734,242 +1686,242 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 -0 val_0 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 26 val_26 2008-04-08 0 -77 val_77 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 77 val_77 2008-04-08 0 118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -338 val_338 2008-04-08 0 -338 val_338 2008-04-08 0 -389 val_389 2008-04-08 0 +224 val_224 2008-04-08 0 389 val_389 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -95 val_95 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 95 val_95 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 143 val_143 2008-04-08 0 -143 val_143 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -190 val_190 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 190 val_190 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -400 val_400 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 +480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 400 val_400 2008-04-08 0 +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 437 val_437 2008-04-08 0 -437 val_437 2008-04-08 0 -448 val_448 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -187 val_187 2008-04-08 0 187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 224 val_224 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -477 val_477 2008-04-08 0 -477 val_477 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 480 val_480 2008-04-08 0 -495 val_495 2008-04-08 0 -495 val_495 2008-04-08 0 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 37 val_37 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 -125 val_125 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 125 val_125 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 260 val_260 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 -466 val_466 2008-04-08 1 +217 val_217 2008-04-08 1 466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -11 val_11 2008-04-08 1 -11 val_11 2008-04-08 1 -19 val_19 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 217 val_217 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -444 val_444 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 462 val_462 2008-04-08 1 462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 169 val_169 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -66 val_66 2008-04-08 1 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 66 val_66 2008-04-08 1 -114 val_114 2008-04-08 1 -114 val_114 2008-04-08 1 -136 val_136 2008-04-08 1 -136 val_136 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -459 val_459 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 459 val_459 2008-04-08 1 -150 val_150 2008-04-08 1 -150 val_150 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 176 val_176 2008-04-08 1 -183 val_183 2008-04-08 1 -183 val_183 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 187 val_187 2008-04-08 1 -202 val_202 2008-04-08 1 -202 val_202 2008-04-08 1 -228 val_228 2008-04-08 1 -228 val_228 2008-04-08 1 -235 val_235 2008-04-08 1 -235 val_235 2008-04-08 1 -257 val_257 2008-04-08 1 -257 val_257 2008-04-08 1 -309 val_309 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 309 val_309 2008-04-08 1 -374 val_374 2008-04-08 1 -374 val_374 2008-04-08 1 -392 val_392 2008-04-08 1 +327 val_327 2008-04-08 1 392 val_392 2008-04-08 1 396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -491 val_491 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -2002,9 +1954,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2016,29 +1965,16 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) + File Output Operator + compressed: false Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int) + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 295 Data size: 106032 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -2085,478 +2021,478 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@ds=2008-04-08/hr=1 #### A masked pattern was here #### +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -8 val_8 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 -33 val_33 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 33 val_33 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -103 val_103 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 -125 val_125 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 125 val_125 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -275 val_275 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 -305 val_305 2008-04-08 0 +480 val_480 2008-04-08 0 305 val_305 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 -484 val_484 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 0 val_0 2008-04-08 0 -0 val_0 2008-04-08 0 -0 val_0 2008-04-08 0 -0 val_0 2008-04-08 0 -4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 -4 val_4 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -77 val_77 2008-04-08 0 -77 val_77 2008-04-08 0 -77 val_77 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 77 val_77 2008-04-08 0 118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 -158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -158 val_158 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 +480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 327 val_327 2008-04-08 0 338 val_338 2008-04-08 0 -338 val_338 2008-04-08 0 -338 val_338 2008-04-08 0 -338 val_338 2008-04-08 0 -389 val_389 2008-04-08 0 -389 val_389 2008-04-08 0 -389 val_389 2008-04-08 0 -389 val_389 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 -404 val_404 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 +187 val_187 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -419 val_419 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -80 val_80 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -95 val_95 2008-04-08 0 -95 val_95 2008-04-08 0 -95 val_95 2008-04-08 0 +118 val_118 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 95 val_95 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -129 val_129 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -282 val_282 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -367 val_367 2008-04-08 0 -143 val_143 2008-04-08 0 143 val_143 2008-04-08 0 -143 val_143 2008-04-08 0 -143 val_143 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 -190 val_190 2008-04-08 0 -190 val_190 2008-04-08 0 -190 val_190 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 +480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 190 val_190 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -242 val_242 2008-04-08 0 -400 val_400 2008-04-08 0 -400 val_400 2008-04-08 0 -400 val_400 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 +480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 +480 val_480 2008-04-08 0 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 400 val_400 2008-04-08 0 +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 +0 val_0 2008-04-08 0 +316 val_316 2008-04-08 0 +129 val_129 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 437 val_437 2008-04-08 0 -437 val_437 2008-04-08 0 -437 val_437 2008-04-08 0 -437 val_437 2008-04-08 0 -448 val_448 2008-04-08 0 -448 val_448 2008-04-08 0 -448 val_448 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -459 val_459 2008-04-08 0 -187 val_187 2008-04-08 0 -187 val_187 2008-04-08 0 187 val_187 2008-04-08 0 -187 val_187 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 +404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 +404 val_404 2008-04-08 0 +282 val_282 2008-04-08 0 +316 val_316 2008-04-08 0 +77 val_77 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -309 val_309 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -396 val_396 2008-04-08 0 -477 val_477 2008-04-08 0 -477 val_477 2008-04-08 0 -477 val_477 2008-04-08 0 -477 val_477 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 -480 val_480 2008-04-08 0 +389 val_389 2008-04-08 0 +242 val_242 2008-04-08 0 +242 val_242 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 480 val_480 2008-04-08 0 -495 val_495 2008-04-08 0 -495 val_495 2008-04-08 0 -495 val_495 2008-04-08 0 -495 val_495 2008-04-08 0 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 -37 val_37 2008-04-08 1 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 -103 val_103 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 103 val_103 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -125 val_125 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -260 val_260 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 -323 val_323 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 323 val_323 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -411 val_411 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 -455 val_455 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 -466 val_466 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 0 val_0 2008-04-08 1 -11 val_11 2008-04-08 1 -11 val_11 2008-04-08 1 -11 val_11 2008-04-08 1 -11 val_11 2008-04-08 1 -19 val_19 2008-04-08 1 -19 val_19 2008-04-08 1 -19 val_19 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -44 val_44 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 -217 val_217 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 217 val_217 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -327 val_327 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -356 val_356 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -378 val_378 2008-04-08 1 -444 val_444 2008-04-08 1 -444 val_444 2008-04-08 1 -444 val_444 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 462 val_462 2008-04-08 1 462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -95 val_95 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -129 val_129 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 -169 val_169 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 169 val_169 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -282 val_282 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -367 val_367 2008-04-08 1 -66 val_66 2008-04-08 1 -66 val_66 2008-04-08 1 -66 val_66 2008-04-08 1 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 66 val_66 2008-04-08 1 -114 val_114 2008-04-08 1 -114 val_114 2008-04-08 1 -114 val_114 2008-04-08 1 -114 val_114 2008-04-08 1 -136 val_136 2008-04-08 1 -136 val_136 2008-04-08 1 -136 val_136 2008-04-08 1 -136 val_136 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -286 val_286 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -345 val_345 2008-04-08 1 -459 val_459 2008-04-08 1 -459 val_459 2008-04-08 1 -459 val_459 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 459 val_459 2008-04-08 1 -150 val_150 2008-04-08 1 -150 val_150 2008-04-08 1 -150 val_150 2008-04-08 1 -150 val_150 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 -176 val_176 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 176 val_176 2008-04-08 1 -183 val_183 2008-04-08 1 -183 val_183 2008-04-08 1 -183 val_183 2008-04-08 1 -183 val_183 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 187 val_187 2008-04-08 1 -202 val_202 2008-04-08 1 -202 val_202 2008-04-08 1 -202 val_202 2008-04-08 1 -202 val_202 2008-04-08 1 -228 val_228 2008-04-08 1 -228 val_228 2008-04-08 1 -228 val_228 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 +309 val_309 2008-04-08 1 +327 val_327 2008-04-08 1 +392 val_392 2008-04-08 1 +396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 235 val_235 2008-04-08 1 -235 val_235 2008-04-08 1 -235 val_235 2008-04-08 1 -235 val_235 2008-04-08 1 -257 val_257 2008-04-08 1 -257 val_257 2008-04-08 1 -257 val_257 2008-04-08 1 +44 val_44 2008-04-08 1 257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 +367 val_367 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 +125 val_125 2008-04-08 1 +286 val_286 2008-04-08 1 +176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 +176 val_176 2008-04-08 1 +260 val_260 2008-04-08 1 +217 val_217 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 309 val_309 2008-04-08 1 -309 val_309 2008-04-08 1 -309 val_309 2008-04-08 1 -309 val_309 2008-04-08 1 -374 val_374 2008-04-08 1 -374 val_374 2008-04-08 1 -374 val_374 2008-04-08 1 -374 val_374 2008-04-08 1 -392 val_392 2008-04-08 1 -392 val_392 2008-04-08 1 -392 val_392 2008-04-08 1 +327 val_327 2008-04-08 1 392 val_392 2008-04-08 1 396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 -491 val_491 2008-04-08 1 -491 val_491 2008-04-08 1 -491 val_491 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 +462 val_462 2008-04-08 1 +462 val_462 2008-04-08 1 +183 val_183 2008-04-08 1 +37 val_37 2008-04-08 1 +194 val_194 2008-04-08 1 +169 val_169 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 @@ -2589,9 +2525,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -2603,29 +2536,16 @@ STAGE PLANS: expressions: key (type: int), value (type: string), ds (type: string), hr (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - value expressions: _col2 (type: string), _col3 (type: int) + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcbucket_mapjoin_n8 Execution mode: vectorized, llap LLAP IO: no inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: int) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 73 Data size: 26696 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.srcbucket_mapjoin_n8 Stage: Stage-2 Dependency Collection @@ -2673,124 +2593,124 @@ POSTHOOK: Input: default@srcbucket_mapjoin_n8 POSTHOOK: Input: default@srcbucket_mapjoin_n8@hr=0 POSTHOOK: Input: default@srcbucket_mapjoin_n8@hr=1 #### A masked pattern was here #### +484 val_484 2008-04-08 0 +224 val_224 2008-04-08 0 +213 val_213 2008-04-08 0 +495 val_495 2008-04-08 0 +327 val_327 2008-04-08 0 +338 val_338 2008-04-08 0 +466 val_466 2008-04-08 0 +309 val_309 2008-04-08 0 0 val_0 2008-04-08 0 -26 val_26 2008-04-08 0 -26 val_26 2008-04-08 0 -33 val_33 2008-04-08 0 -84 val_84 2008-04-08 0 -84 val_84 2008-04-08 0 -103 val_103 2008-04-08 0 -118 val_118 2008-04-08 0 -118 val_118 2008-04-08 0 +316 val_316 2008-04-08 0 129 val_129 2008-04-08 0 -158 val_158 2008-04-08 0 +4 val_4 2008-04-08 0 +169 val_169 2008-04-08 0 +437 val_437 2008-04-08 0 187 val_187 2008-04-08 0 -190 val_190 2008-04-08 0 -338 val_338 2008-04-08 0 +459 val_459 2008-04-08 0 +103 val_103 2008-04-08 0 +213 val_213 2008-04-08 0 +275 val_275 2008-04-08 0 404 val_404 2008-04-08 0 +84 val_84 2008-04-08 0 +8 val_8 2008-04-08 0 +172 val_172 2008-04-08 0 +158 val_158 2008-04-08 0 +26 val_26 2008-04-08 0 +327 val_327 2008-04-08 0 404 val_404 2008-04-08 0 -437 val_437 2008-04-08 0 -466 val_466 2008-04-08 0 -466 val_466 2008-04-08 0 -477 val_477 2008-04-08 0 -4 val_4 2008-04-08 0 -80 val_80 2008-04-08 0 -95 val_95 2008-04-08 0 -143 val_143 2008-04-08 0 -275 val_275 2008-04-08 0 -316 val_316 2008-04-08 0 -316 val_316 2008-04-08 0 +282 val_282 2008-04-08 0 316 val_316 2008-04-08 0 -367 val_367 2008-04-08 0 77 val_77 2008-04-08 0 -125 val_125 2008-04-08 0 -172 val_172 2008-04-08 0 -172 val_172 2008-04-08 0 +118 val_118 2008-04-08 0 +419 val_419 2008-04-08 0 +118 val_118 2008-04-08 0 +224 val_224 2008-04-08 0 +389 val_389 2008-04-08 0 242 val_242 2008-04-08 0 242 val_242 2008-04-08 0 -389 val_389 2008-04-08 0 -448 val_448 2008-04-08 0 -459 val_459 2008-04-08 0 -484 val_484 2008-04-08 0 -495 val_495 2008-04-08 0 -8 val_8 2008-04-08 0 -224 val_224 2008-04-08 0 -224 val_224 2008-04-08 0 -305 val_305 2008-04-08 0 -309 val_309 2008-04-08 0 -327 val_327 2008-04-08 0 -327 val_327 2008-04-08 0 -341 val_341 2008-04-08 0 -396 val_396 2008-04-08 0 -419 val_419 2008-04-08 0 -169 val_169 2008-04-08 0 -169 val_169 2008-04-08 0 -213 val_213 2008-04-08 0 -213 val_213 2008-04-08 0 -282 val_282 2008-04-08 0 -400 val_400 2008-04-08 0 +95 val_95 2008-04-08 0 +143 val_143 2008-04-08 0 +33 val_33 2008-04-08 0 +367 val_367 2008-04-08 0 480 val_480 2008-04-08 0 +316 val_316 2008-04-08 0 +80 val_80 2008-04-08 0 +466 val_466 2008-04-08 0 +190 val_190 2008-04-08 0 +396 val_396 2008-04-08 0 +125 val_125 2008-04-08 0 480 val_480 2008-04-08 0 +305 val_305 2008-04-08 0 +169 val_169 2008-04-08 0 480 val_480 2008-04-08 0 -0 val_0 2008-04-08 1 -0 val_0 2008-04-08 1 -44 val_44 2008-04-08 1 -103 val_103 2008-04-08 1 -129 val_129 2008-04-08 1 -187 val_187 2008-04-08 1 -187 val_187 2008-04-08 1 -194 val_194 2008-04-08 1 -260 val_260 2008-04-08 1 -466 val_466 2008-04-08 1 -95 val_95 2008-04-08 1 -286 val_286 2008-04-08 1 +172 val_172 2008-04-08 0 +26 val_26 2008-04-08 0 +341 val_341 2008-04-08 0 +84 val_84 2008-04-08 0 +448 val_448 2008-04-08 0 +477 val_477 2008-04-08 0 +400 val_400 2008-04-08 0 +165 val_165 2008-04-08 1 +150 val_150 2008-04-08 1 +66 val_66 2008-04-08 1 +374 val_374 2008-04-08 1 +37 val_37 2008-04-08 1 +15 val_15 2008-04-08 1 +459 val_459 2008-04-08 1 +396 val_396 2008-04-08 1 367 val_367 2008-04-08 1 -491 val_491 2008-04-08 1 -114 val_114 2008-04-08 1 +455 val_455 2008-04-08 1 +345 val_345 2008-04-08 1 +378 val_378 2008-04-08 1 +356 val_356 2008-04-08 1 125 val_125 2008-04-08 1 -136 val_136 2008-04-08 1 -165 val_165 2008-04-08 1 -165 val_165 2008-04-08 1 +286 val_286 2008-04-08 1 176 val_176 2008-04-08 1 +51 val_51 2008-04-08 1 +239 val_239 2008-04-08 1 176 val_176 2008-04-08 1 -202 val_202 2008-04-08 1 -217 val_217 2008-04-08 1 +260 val_260 2008-04-08 1 217 val_217 2008-04-08 1 -323 val_323 2008-04-08 1 -455 val_455 2008-04-08 1 -459 val_459 2008-04-08 1 -11 val_11 2008-04-08 1 -66 val_66 2008-04-08 1 -150 val_150 2008-04-08 1 -183 val_183 2008-04-08 1 -235 val_235 2008-04-08 1 -239 val_239 2008-04-08 1 -239 val_239 2008-04-08 1 +466 val_466 2008-04-08 1 +411 val_411 2008-04-08 1 +129 val_129 2008-04-08 1 +0 val_0 2008-04-08 1 +165 val_165 2008-04-08 1 +51 val_51 2008-04-08 1 +95 val_95 2008-04-08 1 +187 val_187 2008-04-08 1 +169 val_169 2008-04-08 1 +0 val_0 2008-04-08 1 +282 val_282 2008-04-08 1 +15 val_15 2008-04-08 1 +19 val_19 2008-04-08 1 309 val_309 2008-04-08 1 327 val_327 2008-04-08 1 -345 val_345 2008-04-08 1 392 val_392 2008-04-08 1 396 val_396 2008-04-08 1 -396 val_396 2008-04-08 1 +11 val_11 2008-04-08 1 +228 val_228 2008-04-08 1 +103 val_103 2008-04-08 1 +239 val_239 2008-04-08 1 +202 val_202 2008-04-08 1 +235 val_235 2008-04-08 1 +44 val_44 2008-04-08 1 +257 val_257 2008-04-08 1 +114 val_114 2008-04-08 1 +217 val_217 2008-04-08 1 +187 val_187 2008-04-08 1 +491 val_491 2008-04-08 1 +444 val_444 2008-04-08 1 +323 val_323 2008-04-08 1 +136 val_136 2008-04-08 1 462 val_462 2008-04-08 1 462 val_462 2008-04-08 1 -15 val_15 2008-04-08 1 -15 val_15 2008-04-08 1 -19 val_19 2008-04-08 1 -37 val_37 2008-04-08 1 +183 val_183 2008-04-08 1 37 val_37 2008-04-08 1 -51 val_51 2008-04-08 1 -51 val_51 2008-04-08 1 -169 val_169 2008-04-08 1 +194 val_194 2008-04-08 1 169 val_169 2008-04-08 1 -228 val_228 2008-04-08 1 -257 val_257 2008-04-08 1 -282 val_282 2008-04-08 1 -356 val_356 2008-04-08 1 -374 val_374 2008-04-08 1 -378 val_378 2008-04-08 1 -411 val_411 2008-04-08 1 -444 val_444 2008-04-08 1 PREHOOK: query: drop table srcbucket_mapjoin_n8 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcbucket_mapjoin_n8 diff --git a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out index c024062c74..9255be7dce 100644 --- a/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out +++ b/ql/src/test/results/clientpositive/llap/load_dyn_part5.q.out @@ -52,52 +52,28 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part5 - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: key, value - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll') - keys: value (type: string) - mode: hash - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col1 (type: struct) - Execution mode: llap + Reduce Output Operator + key expressions: _col1 (type: string) + sort order: + + Map-reduce partition columns: _col1 (type: string) + value expressions: _col0 (type: string) + Execution mode: vectorized, llap LLAP IO: no inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial + Select Operator + expressions: VALUE._col0 (type: string), KEY._col1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col1 (type: struct), _col0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 132750 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part5 Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out index 2c838b6a15..c48810cb5a 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out @@ -38,9 +38,6 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-1 Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -56,30 +53,16 @@ STAGE PLANS: expressions: value (type: string), key (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - sort order: + - Map-reduce partition columns: _col1 (type: string) + File Output Operator + compressed: false Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.partition_mv_sdp Execution mode: vectorized, llap LLAP IO: may be used (ACID table) - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), KEY._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Dp Sort State: PARTITION_SORTED - Statistics: Num rows: 55 Data size: 19268 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.partition_mv_sdp Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/llap/mm_dp.q.out b/ql/src/test/results/clientpositive/llap/mm_dp.q.out index b23a3d720b..6ad05e9ed1 100644 --- a/ql/src/test/results/clientpositive/llap/mm_dp.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_dp.q.out @@ -149,7 +149,6 @@ STAGE PLANS: key expressions: _col3 (type: string), _bucket_number (type: string), _col2 (type: int) sort order: +++ Map-reduce partition columns: _col3 (type: string) - Statistics: Num rows: 2605 Data size: 268315 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col1 (type: int) Execution mode: vectorized, llap LLAP IO: all inputs @@ -159,11 +158,10 @@ STAGE PLANS: Select Operator expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), KEY._col2 (type: int), KEY._col3 (type: string), KEY._bucket_number (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _bucket_number - Statistics: Num rows: 2605 Data size: 747635 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false Dp Sort State: PARTITION_BUCKET_SORTED - Statistics: Num rows: 2605 Data size: 747635 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2605 Data size: 268315 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out index 25426bf5bf..aa45342ddf 100644 --- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out @@ -728,10 +728,10 @@ Table: orc_create_people_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 4 + numFiles 1 numRows 50 - rawDataSize 21955 - totalSize 5341 + rawDataSize 21950 + totalSize 2123 #### A masked pattern was here #### # Storage Information @@ -769,10 +769,10 @@ Table: orc_create_people_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 4 + numFiles 1 numRows 50 - rawDataSize 22043 - totalSize 5339 + rawDataSize 22050 + totalSize 2134 #### A masked pattern was here #### # Storage Information @@ -822,10 +822,10 @@ Table: orc_create_people_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 4 + numFiles 1 numRows 50 - rawDataSize 21955 - totalSize 5341 + rawDataSize 21950 + totalSize 2123 #### A masked pattern was here #### # Storage Information @@ -863,10 +863,10 @@ Table: orc_create_people_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 4 + numFiles 1 numRows 50 - rawDataSize 22043 - totalSize 5339 + rawDataSize 22050 + totalSize 2134 #### A masked pattern was here #### # Storage Information @@ -965,10 +965,10 @@ Table: orc_create_people_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"address\":\"true\",\"first_name\":\"true\",\"id\":\"true\",\"last_name\":\"true\",\"salary\":\"true\",\"start_date\":\"true\"}} - numFiles 4 + numFiles 1 numRows 50 - rawDataSize 21955 - totalSize 5341 + rawDataSize 21950 + totalSize 2123 #### A masked pattern was here #### # Storage Information @@ -1006,10 +1006,10 @@ Table: orc_create_people_n0 #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"address\":\"true\",\"first_name\":\"true\",\"id\":\"true\",\"last_name\":\"true\",\"salary\":\"true\",\"start_date\":\"true\"}} - numFiles 4 + numFiles 1 numRows 50 - rawDataSize 22043 - totalSize 5339 + rawDataSize 22050 + totalSize 2134 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out index d4f95176ab..5bb6432d72 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -76,52 +76,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n1 Stage: Stage-2 Dependency Collection @@ -164,13 +140,8 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 6 items --rw-r--r-- 3 ### USER ### ### GROUP ### 555 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 562 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 561 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 496 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 554 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 478 ### HDFS DATE ### hdfs://### HDFS PATH ### +Found 1 items +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1359 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -212,52 +183,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n1 Stage: Stage-8 Conditional Operator @@ -346,7 +293,7 @@ POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1360 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1359 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -388,52 +335,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n1 Stage: Stage-8 Conditional Operator @@ -514,7 +437,7 @@ POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2461 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1359 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1_n1 WHERE ds='1' diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 95b352a1b3..d943b40374 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -76,52 +76,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1 Stage: Stage-2 Dependency Collection @@ -164,10 +140,8 @@ POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).key EXPRESSION [(src)sr POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 3 items --rw-r--r-- 3 ### USER ### ### GROUP ### 947 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 875 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-r--r-- 3 ### USER ### ### GROUP ### 856 ### HDFS DATE ### hdfs://### HDFS PATH ### +Found 1 items +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1754 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -209,52 +183,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b Stage: Stage-8 Conditional Operator @@ -385,52 +335,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c Stage: Stage-8 Conditional Operator @@ -511,7 +437,7 @@ POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2409 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1754 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1 WHERE ds='1' @@ -618,7 +544,7 @@ POSTHOOK: type: ALTER_PARTITION_MERGE POSTHOOK: Input: default@orcfile_merge1 POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0 Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2409 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1754 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1c WHERE ds='1' @@ -698,80 +624,41 @@ Type: struct Stripe Statistics: Stripe 1: - Column 0: count: 90 hasNull: false - Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736 - Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612 - Stripe 2: - Column 0: count: 78 hasNull: false - Column 1: count: 78 hasNull: false bytesOnDisk: 161 min: 0 max: 497 sum: 18371 - Column 2: count: 78 hasNull: false bytesOnDisk: 380 min: val_0 max: val_95 sum: 529 - Stripe 3: - Column 0: count: 74 hasNull: false - Column 1: count: 74 hasNull: false bytesOnDisk: 153 min: 2 max: 493 sum: 19663 - Column 2: count: 74 hasNull: false bytesOnDisk: 363 min: val_105 max: val_97 sum: 505 + Column 0: count: 242 hasNull: false + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 File Statistics: Column 0: count: 242 hasNull: false - Column 1: count: 242 hasNull: false bytesOnDisk: 499 min: 0 max: 497 sum: 60770 - Column 2: count: 242 hasNull: false bytesOnDisk: 1171 min: val_0 max: val_97 sum: 1646 + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 Stripes: - Stripe: offset: 3 data: 613 rows: 90 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 3 length 11 - Stream: column 1 section ROW_INDEX start: 14 length 27 - Stream: column 2 section ROW_INDEX start: 41 length 38 - Stream: column 1 section DATA start: 79 length 185 - Stream: column 2 section DATA start: 264 length 377 - Stream: column 2 section LENGTH start: 641 length 51 + Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77 + Stream: column 0 section ROW_INDEX start: 3 length 12 + Stream: column 1 section ROW_INDEX start: 15 length 28 + Stream: column 2 section ROW_INDEX start: 43 length 37 + Stream: column 1 section DATA start: 80 length 489 + Stream: column 2 section DATA start: 569 length 247 + Stream: column 2 section LENGTH start: 816 length 71 + Stream: column 2 section DICTIONARY_DATA start: 887 length 592 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 + Encoding column 2: DICTIONARY_V2[153] Row group indices for column 0: - Entry 0: count: 90 hasNull: false positions: + Entry 0: count: 242 hasNull: false positions: Row group indices for column 1: - Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0 + Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0 Row group indices for column 2: - Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0 - Stripe: offset: 753 data: 541 rows: 78 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 753 length 11 - Stream: column 1 section ROW_INDEX start: 764 length 27 - Stream: column 2 section ROW_INDEX start: 791 length 38 - Stream: column 1 section DATA start: 829 length 161 - Stream: column 2 section DATA start: 990 length 332 - Stream: column 2 section LENGTH start: 1322 length 48 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 78 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 78 hasNull: false min: val_0 max: val_95 sum: 529 positions: 0,0,0,0,0 - Stripe: offset: 1431 data: 516 rows: 74 tail: 61 index: 78 - Stream: column 0 section ROW_INDEX start: 1431 length 11 - Stream: column 1 section ROW_INDEX start: 1442 length 27 - Stream: column 2 section ROW_INDEX start: 1469 length 40 - Stream: column 1 section DATA start: 1509 length 153 - Stream: column 2 section DATA start: 1662 length 331 - Stream: column 2 section LENGTH start: 1993 length 32 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 74 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 74 hasNull: false min: val_105 max: val_97 sum: 505 positions: 0,0,0,0,0 + Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 2409 bytes +File length: 1754 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ -- END ORC FILE DUMP -- -86 val_86 1 0 +2 val_2 1 0 PREHOOK: query: select * from orcfile_merge1c where ds='1' and part='0' limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@orcfile_merge1c @@ -787,80 +674,41 @@ Type: struct Stripe Statistics: Stripe 1: - Column 0: count: 90 hasNull: false - Column 1: count: 90 hasNull: false bytesOnDisk: 185 min: 0 max: 495 sum: 22736 - Column 2: count: 90 hasNull: false bytesOnDisk: 428 min: val_0 max: val_86 sum: 612 - Stripe 2: - Column 0: count: 78 hasNull: false - Column 1: count: 78 hasNull: false bytesOnDisk: 161 min: 0 max: 497 sum: 18371 - Column 2: count: 78 hasNull: false bytesOnDisk: 380 min: val_0 max: val_95 sum: 529 - Stripe 3: - Column 0: count: 74 hasNull: false - Column 1: count: 74 hasNull: false bytesOnDisk: 153 min: 2 max: 493 sum: 19663 - Column 2: count: 74 hasNull: false bytesOnDisk: 363 min: val_105 max: val_97 sum: 505 + Column 0: count: 242 hasNull: false + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 File Statistics: Column 0: count: 242 hasNull: false - Column 1: count: 242 hasNull: false bytesOnDisk: 499 min: 0 max: 497 sum: 60770 - Column 2: count: 242 hasNull: false bytesOnDisk: 1171 min: val_0 max: val_97 sum: 1646 + Column 1: count: 242 hasNull: false bytesOnDisk: 489 min: 0 max: 497 sum: 60770 + Column 2: count: 242 hasNull: false bytesOnDisk: 910 min: val_0 max: val_97 sum: 1646 Stripes: - Stripe: offset: 3 data: 613 rows: 90 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 3 length 11 - Stream: column 1 section ROW_INDEX start: 14 length 27 - Stream: column 2 section ROW_INDEX start: 41 length 38 - Stream: column 1 section DATA start: 79 length 185 - Stream: column 2 section DATA start: 264 length 377 - Stream: column 2 section LENGTH start: 641 length 51 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 90 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 90 hasNull: false min: 0 max: 495 sum: 22736 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 90 hasNull: false min: val_0 max: val_86 sum: 612 positions: 0,0,0,0,0 - Stripe: offset: 753 data: 541 rows: 78 tail: 61 index: 76 - Stream: column 0 section ROW_INDEX start: 753 length 11 - Stream: column 1 section ROW_INDEX start: 764 length 27 - Stream: column 2 section ROW_INDEX start: 791 length 38 - Stream: column 1 section DATA start: 829 length 161 - Stream: column 2 section DATA start: 990 length 332 - Stream: column 2 section LENGTH start: 1322 length 48 - Encoding column 0: DIRECT - Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 - Row group indices for column 0: - Entry 0: count: 78 hasNull: false positions: - Row group indices for column 1: - Entry 0: count: 78 hasNull: false min: 0 max: 497 sum: 18371 positions: 0,0,0 - Row group indices for column 2: - Entry 0: count: 78 hasNull: false min: val_0 max: val_95 sum: 529 positions: 0,0,0,0,0 - Stripe: offset: 1431 data: 516 rows: 74 tail: 61 index: 78 - Stream: column 0 section ROW_INDEX start: 1431 length 11 - Stream: column 1 section ROW_INDEX start: 1442 length 27 - Stream: column 2 section ROW_INDEX start: 1469 length 40 - Stream: column 1 section DATA start: 1509 length 153 - Stream: column 2 section DATA start: 1662 length 331 - Stream: column 2 section LENGTH start: 1993 length 32 + Stripe: offset: 3 data: 1399 rows: 242 tail: 73 index: 77 + Stream: column 0 section ROW_INDEX start: 3 length 12 + Stream: column 1 section ROW_INDEX start: 15 length 28 + Stream: column 2 section ROW_INDEX start: 43 length 37 + Stream: column 1 section DATA start: 80 length 489 + Stream: column 2 section DATA start: 569 length 247 + Stream: column 2 section LENGTH start: 816 length 71 + Stream: column 2 section DICTIONARY_DATA start: 887 length 592 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 - Encoding column 2: DIRECT_V2 + Encoding column 2: DICTIONARY_V2[153] Row group indices for column 0: - Entry 0: count: 74 hasNull: false positions: + Entry 0: count: 242 hasNull: false positions: Row group indices for column 1: - Entry 0: count: 74 hasNull: false min: 2 max: 493 sum: 19663 positions: 0,0,0 + Entry 0: count: 242 hasNull: false min: 0 max: 497 sum: 60770 positions: 0,0,0 Row group indices for column 2: - Entry 0: count: 74 hasNull: false min: val_105 max: val_97 sum: 505 positions: 0,0,0,0,0 + Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 2409 bytes +File length: 1754 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ -- END ORC FILE DUMP -- -86 val_86 1 0 +2 val_2 1 0 PREHOOK: query: DROP TABLE orcfile_merge1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@orcfile_merge1 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out index 430ce16239..7265292d09 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -50,52 +50,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 10) (type: int), (hash(value) pmod 10) (type: int) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge2a_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string), CAST( _col3 AS STRING) (type: string) - outputColumnNames: key, value, one, two, three - Statistics: Num rows: 500 Data size: 274000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: one (type: string), two (type: string), three (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) - sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) - Statistics: Num rows: 500 Data size: 658500 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col3 (type: struct), _col4 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int), _col3 (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: int), _col3 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col3 (type: struct), _col4 (type: struct), _col0 (type: string), _col1 (type: string), _col2 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 666500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 51500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge2a_n0 Stage: Stage-2 Dependency Collection @@ -196,7 +172,7 @@ POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=1).value SIMPL POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-r--r-- 3 ### USER ### ### GROUP ### 349 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 349 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a_n0 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out index 7cd5ba4210..a912216dc5 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge_diff_fs.q.out @@ -76,52 +76,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1_n0 Stage: Stage-2 Dependency Collection @@ -164,7 +140,7 @@ POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).key EXPRESSION [(src POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=0).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n0 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Found 6 items +Found 1 items #### A masked pattern was here #### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n0 PARTITION (ds='1', part) @@ -207,52 +183,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1b_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1b_n0 Stage: Stage-8 Conditional Operator @@ -383,52 +335,28 @@ STAGE PLANS: expressions: UDFToInteger(key) (type: int), value (type: string), (hash(key) pmod 2) (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat - serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde - name: default.orcfile_merge1c_n0 - Select Operator - expressions: _col0 (type: int), _col1 (type: string), '1' (type: string), CAST( _col2 AS STRING) (type: string) - outputColumnNames: key, value, ds, part - Statistics: Num rows: 500 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll') - keys: ds (type: string), part (type: string) - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) - Statistics: Num rows: 250 Data size: 283250 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col2 (type: struct), _col3 (type: struct) + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) + value expressions: _col0 (type: int), _col1 (type: string) Execution mode: llap LLAP IO: no inputs Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - aggregations: compute_stats(VALUE._col0), compute_stats(VALUE._col1) - keys: KEY._col0 (type: string), KEY._col1 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col2 (type: struct), _col3 (type: struct), _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 250 Data size: 287250 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), KEY._col2 (type: int) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + Dp Sort State: PARTITION_SORTED + Statistics: Num rows: 500 Data size: 49500 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat + serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde + name: default.orcfile_merge1c_n0 Stage: Stage-8 Conditional Operator diff --git a/ql/src/test/results/clientpositive/llap/sample10.q.out b/ql/src/test/results/clientpositive/llap/sample10.q.out index 05e132cffe..07a5757bd7 100644 --- a/ql/src/test/results/clientpositive/llap/sample10.q.out +++ b/ql/src/test/results/clientpositive/llap/sample10.q.out @@ -69,29 +69,29 @@ STAGE PLANS: TableScan alias: srcpartbucket filterExpr: ds is not null (type: boolean) - Statistics: Num rows: 40 Data size: 10760 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 40 Data size: 14776 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 1) (type: boolean) sampleDesc: BUCKET 2 OUT OF 4 - Statistics: Num rows: 20 Data size: 5380 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 20 Data size: 5380 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7360 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count(1) keys: ds (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: a sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL tag: -1 value expressions: _col1 (type: bigint) auto parallelism: true @@ -118,7 +118,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.srcpartbucket - numFiles 3 + numFiles 1 numRows 10 partition_columns ds/hr partition_columns.types string:string @@ -126,7 +126,7 @@ STAGE PLANS: serialization.ddl struct srcpartbucket { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 295 + totalSize 139 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -169,7 +169,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.srcpartbucket - numFiles 3 + numFiles 1 numRows 10 partition_columns ds/hr partition_columns.types string:string @@ -177,7 +177,7 @@ STAGE PLANS: serialization.ddl struct srcpartbucket { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 295 + totalSize 139 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -220,7 +220,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.srcpartbucket - numFiles 3 + numFiles 1 numRows 10 partition_columns ds/hr partition_columns.types string:string @@ -228,7 +228,7 @@ STAGE PLANS: serialization.ddl struct srcpartbucket { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 295 + totalSize 139 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -271,7 +271,7 @@ STAGE PLANS: columns.types string:string #### A masked pattern was here #### name default.srcpartbucket - numFiles 3 + numFiles 1 numRows 10 partition_columns ds/hr partition_columns.types string:string @@ -279,7 +279,7 @@ STAGE PLANS: serialization.ddl struct srcpartbucket { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe - totalSize 295 + totalSize 139 #### A masked pattern was here #### serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe @@ -318,12 +318,12 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col0 (type: string) null sort order: a sort order: + - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL tag: -1 value expressions: _col1 (type: bigint) auto parallelism: false @@ -334,13 +334,13 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 384 Basic stats: COMPLETE Column stats: PARTIAL #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat diff --git a/ql/src/test/results/clientpositive/llap/sample10_mm.q.out b/ql/src/test/results/clientpositive/llap/sample10_mm.q.out index fb4c6d3a47..a4621dc049 100644 --- a/ql/src/test/results/clientpositive/llap/sample10_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/sample10_mm.q.out @@ -49,46 +49,46 @@ POSTHOOK: Input: default@srcpartbucket@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=12 #### A masked pattern was here #### -5 val_5 2008-04-08 11 -2 val_2 2008-04-08 11 -5 val_5 2008-04-08 11 -5 val_5 2008-04-08 11 0 val_0 2008-04-08 11 +4 val_4 2008-04-08 11 +8 val_8 2008-04-08 11 0 val_0 2008-04-08 11 0 val_0 2008-04-08 11 +5 val_5 2008-04-08 11 +5 val_5 2008-04-08 11 +2 val_2 2008-04-08 11 +5 val_5 2008-04-08 11 9 val_9 2008-04-08 11 -8 val_8 2008-04-08 11 -4 val_4 2008-04-08 11 -5 val_5 2008-04-08 12 -2 val_2 2008-04-08 12 -5 val_5 2008-04-08 12 -5 val_5 2008-04-08 12 0 val_0 2008-04-08 12 +4 val_4 2008-04-08 12 +8 val_8 2008-04-08 12 0 val_0 2008-04-08 12 0 val_0 2008-04-08 12 +5 val_5 2008-04-08 12 +5 val_5 2008-04-08 12 +2 val_2 2008-04-08 12 +5 val_5 2008-04-08 12 9 val_9 2008-04-08 12 -8 val_8 2008-04-08 12 -4 val_4 2008-04-08 12 -5 val_5 2008-04-09 11 -2 val_2 2008-04-09 11 -5 val_5 2008-04-09 11 -5 val_5 2008-04-09 11 0 val_0 2008-04-09 11 +4 val_4 2008-04-09 11 +8 val_8 2008-04-09 11 0 val_0 2008-04-09 11 0 val_0 2008-04-09 11 +5 val_5 2008-04-09 11 +5 val_5 2008-04-09 11 +2 val_2 2008-04-09 11 +5 val_5 2008-04-09 11 9 val_9 2008-04-09 11 -8 val_8 2008-04-09 11 -4 val_4 2008-04-09 11 -5 val_5 2008-04-09 12 -2 val_2 2008-04-09 12 -5 val_5 2008-04-09 12 -5 val_5 2008-04-09 12 0 val_0 2008-04-09 12 +4 val_4 2008-04-09 12 +8 val_8 2008-04-09 12 0 val_0 2008-04-09 12 0 val_0 2008-04-09 12 +5 val_5 2008-04-09 12 +5 val_5 2008-04-09 12 +2 val_2 2008-04-09 12 +5 val_5 2008-04-09 12 9 val_9 2008-04-09 12 -8 val_8 2008-04-09 12 -4 val_4 2008-04-09 12 PREHOOK: query: explain select key from srcpartbucket tablesample (bucket 2 out of 4 on key) PREHOOK: type: QUERY PREHOOK: Input: default@srcpartbucket @@ -154,34 +154,34 @@ POSTHOOK: Input: default@srcpartbucket@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=12 #### A masked pattern was here #### -5 -2 -5 -5 0 0 0 5 -2 5 +2 5 0 0 0 5 -2 5 +2 5 0 0 0 5 -2 5 +2 5 0 0 0 +5 +5 +2 +5 PREHOOK: query: select key from srcpartbucket tablesample (bucket 3 out of 4 on key) PREHOOK: type: QUERY PREHOOK: Input: default@srcpartbucket @@ -198,7 +198,6 @@ POSTHOOK: Input: default@srcpartbucket@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=12 #### A masked pattern was here #### -9 8 9 8 @@ -206,6 +205,7 @@ POSTHOOK: Input: default@srcpartbucket@ds=2008-04-09/hr=12 8 9 8 +9 PREHOOK: query: select key from srcpartbucket tablesample (bucket 4 out of 4 on key) PREHOOK: type: QUERY PREHOOK: Input: default@srcpartbucket @@ -260,20 +260,20 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpartbucket - Statistics: Num rows: 40 Data size: 3400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 40 Data size: 14136 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (((hash(key) & 2147483647) % 4) = 1) (type: boolean) - Statistics: Num rows: 20 Data size: 1700 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7068 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: key (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7068 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 20 Data size: 7068 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Reducer 2 @@ -283,10 +283,10 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3534 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 6 Data size: 510 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 3534 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out b/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out index 8f4f0551cd..4d4d9f8184 100644 --- a/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_input_counters.q.out @@ -1813,21 +1813,17 @@ PREHOOK: Output: default@testpart1 Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_DYNAMIC_PARTITIONS: 74 - CREATED_FILES: 78 + CREATED_FILES: 74 DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 240 - RECORDS_OUT_0: 74 RECORDS_OUT_1_default.testpart1: 240 RECORDS_OUT_INTERMEDIATE_Map_1: 240 RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 - RECORDS_OUT_OPERATOR_FS_12: 240 - RECORDS_OUT_OPERATOR_FS_9: 74 - RECORDS_OUT_OPERATOR_GBY_7: 74 + RECORDS_OUT_OPERATOR_FS_16: 240 RECORDS_OUT_OPERATOR_MAP_0: 0 RECORDS_OUT_OPERATOR_RS_14: 240 - RECORDS_OUT_OPERATOR_SEL_11: 148 RECORDS_OUT_OPERATOR_SEL_13: 148 - RECORDS_OUT_OPERATOR_SEL_8: 74 + RECORDS_OUT_OPERATOR_SEL_15: 74 RECORDS_OUT_OPERATOR_TS_0: 148 TOTAL_TABLE_ROWS_WRITTEN: 240 Stage-1 LLAP IO COUNTERS: @@ -2246,31 +2242,31 @@ STAGE PLANS: TableScan alias: t1 filterExpr: v is not null (type: boolean) - Statistics: Num rows: 1000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 190616 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: k (type: int), v (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 187804 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 187804 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int) Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 188000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 187804 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator keys: _col0 (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 93904 Basic stats: COMPLETE Column stats: PARTIAL Dynamic Partitioning Event Operator Target column: v (string) Target Input: t2 Partition key expr: v - Statistics: Num rows: 500 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 93904 Basic stats: COMPLETE Column stats: PARTIAL Target Vertex: Map 4 Execution mode: vectorized, llap LLAP IO: no inputs @@ -2279,16 +2275,16 @@ STAGE PLANS: TableScan alias: t2 filterExpr: v is not null (type: boolean) - Statistics: Num rows: 240 Data size: 45120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 240 Data size: 45758 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: k (type: int), v (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 240 Data size: 45120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 240 Data size: 45076 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 240 Data size: 45120 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 240 Data size: 45076 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int) Execution mode: vectorized, llap LLAP IO: no inputs @@ -2302,14 +2298,14 @@ STAGE PLANS: 0 _col1 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col2 - Statistics: Num rows: 776 Data size: 6208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 776 Data size: 5968 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: hash(_col0,_col2) (type: int) outputColumnNames: _col0 - Statistics: Num rows: 776 Data size: 6208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 776 Data size: 5968 Basic stats: COMPLETE Column stats: PARTIAL Reduce Output Operator sort order: - Statistics: Num rows: 776 Data size: 6208 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 776 Data size: 5968 Basic stats: COMPLETE Column stats: PARTIAL value expressions: _col0 (type: int) Reducer 3 Execution mode: vectorized, llap @@ -2318,10 +2314,10 @@ STAGE PLANS: aggregations: sum(VALUE._col0) mode: complete outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: PARTIAL table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat