diff --git ql/src/test/queries/clientnegative/join_nonexistent_part.q ql/src/test/queries/clientnegative/join_nonexistent_part.q index b545a87..dea3406 100644 --- ql/src/test/queries/clientnegative/join_nonexistent_part.q +++ ql/src/test/queries/clientnegative/join_nonexistent_part.q @@ -1,5 +1,4 @@ -set hive.cbo.enable=false; SET hive.security.authorization.enabled = true; SELECT * FROM srcpart s1 join src s2 on s1.key == s2.key -WHERE s1.ds='non-existent'; \ No newline at end of file +WHERE s1.ds='non-existent'; diff --git ql/src/test/queries/clientnegative/joinneg.q ql/src/test/queries/clientnegative/joinneg.q index 9fea99e..a4967fd 100644 --- ql/src/test/queries/clientnegative/joinneg.q +++ ql/src/test/queries/clientnegative/joinneg.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; EXPLAIN FROM (SELECT src.* FROM src) x JOIN diff --git ql/src/test/queries/clientpositive/annotate_stats_groupby.q ql/src/test/queries/clientpositive/annotate_stats_groupby.q index 30fc0f8..854e401 100644 --- ql/src/test/queries/clientpositive/annotate_stats_groupby.q +++ ql/src/test/queries/clientpositive/annotate_stats_groupby.q @@ -1,7 +1,5 @@ set hive.stats.fetch.column.stats=true; set hive.map.aggr.hash.percentmemory=0.0f; -set hive.cbo.enable=false; --- Stats explain output changes with CBO, disable for now. -- hash aggregation is disabled diff --git ql/src/test/queries/clientpositive/annotate_stats_groupby2.q ql/src/test/queries/clientpositive/annotate_stats_groupby2.q index c5b1746..eeac876 100644 --- ql/src/test/queries/clientpositive/annotate_stats_groupby2.q +++ ql/src/test/queries/clientpositive/annotate_stats_groupby2.q @@ -1,6 +1,3 @@ -set hive.cbo.enable=false; --- Stats explain output changes with CBO, disable for now. - drop table location; -- There are different cases for Group By depending on map/reduce side, hash aggregation, @@ -64,4 +61,4 @@ explain select state, country from location group by state, country; -- Case 8: column stats, grouping sets - cardinality = 8 explain select state, country from location group by state, country with cube; -drop table location; \ No newline at end of file +drop table location; diff --git ql/src/test/queries/clientpositive/constprog2.q ql/src/test/queries/clientpositive/constprog2.q index 36e6a22..6001668 100644 --- ql/src/test/queries/clientpositive/constprog2.q +++ ql/src/test/queries/clientpositive/constprog2.q @@ -1,9 +1,5 @@ set hive.fetch.task.conversion=more; set hive.optimize.constant.propagation=true; -set hive.cbo.enable=false; - --- Some constant propagation does not work in CBO due to added casts(?), disable for now. - EXPLAIN SELECT src1.key, src1.key + 1, src2.value diff --git ql/src/test/queries/clientpositive/correlationoptimizer8.q ql/src/test/queries/clientpositive/correlationoptimizer8.q index 8fcb130..4f4c3c5 100644 --- ql/src/test/queries/clientpositive/correlationoptimizer8.q +++ ql/src/test/queries/clientpositive/correlationoptimizer8.q @@ -1,6 +1,5 @@ set hive.auto.convert.join=false; set hive.optimize.correlation=false; -set hive.cbo.enable=false; -- SORT_QUERY_RESULTS diff --git ql/src/test/queries/clientpositive/filter_join_breaktask2.q ql/src/test/queries/clientpositive/filter_join_breaktask2.q index ae2dfba..4c99f77 100644 --- ql/src/test/queries/clientpositive/filter_join_breaktask2.q +++ ql/src/test/queries/clientpositive/filter_join_breaktask2.q @@ -1,6 +1,3 @@ -set hive.cbo.enable=false; - - create table T1(c1 string, c2 string, c3 string, c4 string, c5 string, c6 string, c7 string) partitioned by (ds string); diff --git ql/src/test/queries/clientpositive/infer_bucket_sort.q ql/src/test/queries/clientpositive/infer_bucket_sort.q index 9c0872b..07466fd 100644 --- ql/src/test/queries/clientpositive/infer_bucket_sort.q +++ ql/src/test/queries/clientpositive/infer_bucket_sort.q @@ -1,10 +1,7 @@ set hive.exec.infer.bucket.sort=true; -set hive.cbo.enable=false; - -- This tests inferring how data is bucketed/sorted from the operators in the reducer -- and populating that information in partitions' metadata --- CBO is disabled; it rewrites query plans, messing with the specifically crafted test cases. CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING); diff --git ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q index d40045a..090f04f 100644 --- ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q +++ ql/src/test/queries/clientpositive/infer_bucket_sort_reducers_power_two.q @@ -1,12 +1,10 @@ set hive.exec.infer.bucket.sort=true; set hive.exec.infer.bucket.sort.num.buckets.power.two=true; set hive.exec.reducers.bytes.per.reducer=2500; -set hive.cbo.enable=false; -- This tests inferring how data is bucketed/sorted from the operators in the reducer -- and populating that information in partitions' metadata, it also verifies that the -- number of reducers chosen will be a power of two --- CBO is disabled; it rewrites query plans, messing with the specifically crafted test cases. CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING); diff --git ql/src/test/queries/clientpositive/join12.q ql/src/test/queries/clientpositive/join12.q index b438394..7053a06 100644 --- ql/src/test/queries/clientpositive/join12.q +++ ql/src/test/queries/clientpositive/join12.q @@ -1,7 +1,3 @@ -set hive.cbo.enable=false; - --- Disable CBO for now as it eliminates the n-way join - EXPLAIN SELECT src1.c1, src2.c4 FROM diff --git ql/src/test/queries/clientpositive/list_bucket_dml_1.q ql/src/test/queries/clientpositive/list_bucket_dml_1.q index 7a63493..1062e55 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_1.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_1.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_10.q ql/src/test/queries/clientpositive/list_bucket_dml_10.q index d2848e4..2fbafc2 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_10.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_10.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set mapred.input.dir.recursive=true; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_11.q ql/src/test/queries/clientpositive/list_bucket_dml_11.q index ff03c9b..8260459 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_11.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_11.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set mapred.input.dir.recursive=true; set hive.merge.mapfiles=false; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_12.q ql/src/test/queries/clientpositive/list_bucket_dml_12.q index c8801a4..0cab149 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_12.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_12.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set mapred.input.dir.recursive=true; set hive.merge.mapfiles=false; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_13.q ql/src/test/queries/clientpositive/list_bucket_dml_13.q index f0c0b23..5ea333f 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_13.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_13.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set mapred.input.dir.recursive=true; set hive.merge.mapfiles=false; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_14.q ql/src/test/queries/clientpositive/list_bucket_dml_14.q index 636037b..f60c5b0 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_14.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_14.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_2.q ql/src/test/queries/clientpositive/list_bucket_dml_2.q index 00978e7..9715ff6 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_2.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_2.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_3.q ql/src/test/queries/clientpositive/list_bucket_dml_3.q index d6bb66a..a43613c 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_3.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_3.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_4.q ql/src/test/queries/clientpositive/list_bucket_dml_4.q index 5115054..1ca54fa 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_4.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_4.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_5.q ql/src/test/queries/clientpositive/list_bucket_dml_5.q index 98ea43f..5278a80 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_5.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_5.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_6.q ql/src/test/queries/clientpositive/list_bucket_dml_6.q index 3ea6a5b..3445255 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_6.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_6.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_7.q ql/src/test/queries/clientpositive/list_bucket_dml_7.q index 7946c7b..cacf4ea 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_7.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_7.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_8.q ql/src/test/queries/clientpositive/list_bucket_dml_8.q index 5bf9472..26103f9 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_8.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_8.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_dml_9.q ql/src/test/queries/clientpositive/list_bucket_dml_9.q index 52e81a2..4ddcddd 100644 --- ql/src/test/queries/clientpositive/list_bucket_dml_9.q +++ ql/src/test/queries/clientpositive/list_bucket_dml_9.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q index b6b5867..48b1ff3 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.optimize.listbucketing=true; set mapred.input.dir.recursive=true; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q index 5b8f995..00410d1 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.optimize.listbucketing=true; set mapred.input.dir.recursive=true; diff --git ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q index 21d9db7..2d1193d 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q +++ ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.optimize.listbucketing=true; set mapred.input.dir.recursive=true; diff --git ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q index 094ab41..84727fe 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q +++ ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.optimize.listbucketing=true; set mapred.input.dir.recursive=true; diff --git ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q index e9fecf9..29409e1 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q +++ ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.optimize.listbucketing=true; set mapred.input.dir.recursive=true; diff --git ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q index fbab801..5d69906 100644 --- ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q +++ ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.optimize.listbucketing=true; set mapred.input.dir.recursive=true; set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; diff --git ql/src/test/queries/clientpositive/partition_wise_fileformat2.q ql/src/test/queries/clientpositive/partition_wise_fileformat2.q index bd84ed5..de51cdd 100644 --- ql/src/test/queries/clientpositive/partition_wise_fileformat2.q +++ ql/src/test/queries/clientpositive/partition_wise_fileformat2.q @@ -1,6 +1,3 @@ -set hive.cbo.enable=false; --- Disable CBO since it causes filter pruning to fail to prune in this case - -- SORT_BEFORE_DIFF create table partition_test_partitioned(key string, value string) partitioned by (dt string); diff --git ql/src/test/queries/clientpositive/ppd_join3.q ql/src/test/queries/clientpositive/ppd_join3.q index fef111e..7abf410 100644 --- ql/src/test/queries/clientpositive/ppd_join3.q +++ ql/src/test/queries/clientpositive/ppd_join3.q @@ -1,8 +1,5 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=false; -set hive.cbo.enable=false; - --- Disable CBO for now as it eliminates the n-way join EXPLAIN SELECT src1.c1, src2.c4 diff --git ql/src/test/queries/clientpositive/ppd_outer_join4.q ql/src/test/queries/clientpositive/ppd_outer_join4.q index c5e0003..e465eda 100644 --- ql/src/test/queries/clientpositive/ppd_outer_join4.q +++ ql/src/test/queries/clientpositive/ppd_outer_join4.q @@ -1,8 +1,5 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=false; -set hive.cbo.enable=false; - --- Disable CBO for now as it eliminates the n-way join EXPLAIN FROM diff --git ql/src/test/queries/clientpositive/truncate_column_list_bucket.q ql/src/test/queries/clientpositive/truncate_column_list_bucket.q index 49e0b08..87289f0 100644 --- ql/src/test/queries/clientpositive/truncate_column_list_bucket.q +++ ql/src/test/queries/clientpositive/truncate_column_list_bucket.q @@ -1,4 +1,3 @@ -set hive.cbo.enable=false; set hive.mapred.supports.subdirectories=true; set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; set hive.merge.mapfiles=false; diff --git ql/src/test/results/clientnegative/join_nonexistent_part.q.out ql/src/test/results/clientnegative/join_nonexistent_part.q.out index c2a85aa..a298536 100644 --- ql/src/test/results/clientnegative/join_nonexistent_part.q.out +++ ql/src/test/results/clientnegative/join_nonexistent_part.q.out @@ -1 +1,2 @@ +Warning: Shuffle Join JOIN[8][tables = [$hdt$_0]] in Stage 'Stage-1:MAPRED' is a cross product Authorization failed:No privilege 'Select' found for inputs { database:default, table:src, columnName:key}. Use SHOW GRANT to get more details. diff --git ql/src/test/results/clientnegative/joinneg.q.out ql/src/test/results/clientnegative/joinneg.q.out index 8df43d7..08cfd8e 100644 --- ql/src/test/results/clientnegative/joinneg.q.out +++ ql/src/test/results/clientnegative/joinneg.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10009]: Line 6:12 Invalid table alias 'b' +FAILED: SemanticException [Error 10002]: Line 5:12 Invalid column reference 'b' diff --git ql/src/test/results/clientpositive/annotate_stats_groupby.q.out ql/src/test/results/clientpositive/annotate_stats_groupby.q.out index 51c45c3..4f48aef 100644 --- ql/src/test/results/clientpositive/annotate_stats_groupby.q.out +++ ql/src/test/results/clientpositive/annotate_stats_groupby.q.out @@ -1,6 +1,4 @@ -PREHOOK: query: -- Stats explain output changes with CBO, disable for now. - --- hash aggregation is disabled +PREHOOK: query: -- hash aggregation is disabled -- There are different cases for Group By depending on map/reduce side, hash aggregation, -- grouping sets and column stats. If we don't have column stats, we just assume hash @@ -29,9 +27,7 @@ create table if not exists loc_staging ( PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@loc_staging -POSTHOOK: query: -- Stats explain output changes with CBO, disable for now. - --- hash aggregation is disabled +POSTHOOK: query: -- hash aggregation is disabled -- There are different cases for Group By depending on map/reduce side, hash aggregation, -- grouping sets and column stats. If we don't have column stats, we just assume hash @@ -161,11 +157,11 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator aggregations: count() - keys: state (type: string), locid (type: int) + keys: _col0 (type: string), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 8 Data size: 752 Basic stats: COMPLETE Column stats: PARTIAL @@ -183,12 +179,12 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 7 Data size: 658 Basic stats: COMPLETE Column stats: PARTIAL Select Operator - expressions: _col0 (type: string), _col1 (type: int), _col2 (type: bigint) + expressions: _col0 (type: string), _col2 (type: bigint), _col1 (type: int) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 7 Data size: 658 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator - aggregations: min(_col1) - keys: _col0 (type: string), _col2 (type: bigint) + aggregations: min(_col2) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 7 Data size: 686 Basic stats: COMPLETE Column stats: PARTIAL @@ -263,10 +259,10 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: year (type: int) - outputColumnNames: year + outputColumnNames: _col0 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: year (type: int) + keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Statistics: Num rows: 8 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE @@ -320,10 +316,10 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int) - outputColumnNames: state, locid + outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: state (type: string), locid (type: int) + keys: _col0 (type: string), _col1 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE @@ -723,10 +719,10 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: year (type: int) - outputColumnNames: year + outputColumnNames: _col0 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: year (type: int) + keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Statistics: Num rows: 4 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE @@ -839,10 +835,10 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), zip (type: bigint) - outputColumnNames: state, zip + outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator - keys: state (type: string), zip (type: bigint) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 4 Data size: 344 Basic stats: COMPLETE Column stats: PARTIAL @@ -1238,10 +1234,10 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: year (type: int) - outputColumnNames: year + outputColumnNames: _col0 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: year (type: int) + keys: _col0 (type: int) mode: hash outputColumnNames: _col0 Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out index b84a3fc..bcc5745 100644 --- ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out +++ ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out @@ -1,10 +1,6 @@ -PREHOOK: query: -- Stats explain output changes with CBO, disable for now. - -drop table location +PREHOOK: query: drop table location PREHOOK: type: DROPTABLE -POSTHOOK: query: -- Stats explain output changes with CBO, disable for now. - -drop table location +POSTHOOK: query: drop table location POSTHOOK: type: DROPTABLE PREHOOK: query: -- There are different cases for Group By depending on map/reduce side, hash aggregation, -- grouping sets and column stats. If we don't have column stats, we just assume hash @@ -95,10 +91,10 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: state (type: string), country (type: string) - outputColumnNames: state, country + outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE Group By Operator - keys: state (type: string), country (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE @@ -213,10 +209,10 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), country (type: string) - outputColumnNames: state, country + outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: state (type: string), country (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 1384 Basic stats: COMPLETE Column stats: COMPLETE @@ -272,10 +268,10 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), votes (type: bigint) - outputColumnNames: state, votes + outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: PARTIAL Group By Operator - keys: state (type: string), votes (type: bigint) + keys: _col0 (type: string), _col1 (type: bigint) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 8 Data size: 688 Basic stats: COMPLETE Column stats: PARTIAL @@ -386,10 +382,10 @@ STAGE PLANS: Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), country (type: string) - outputColumnNames: state, country + outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator - keys: state (type: string), country (type: string) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 20 Data size: 3460 Basic stats: COMPLETE Column stats: COMPLETE diff --git ql/src/test/results/clientpositive/constprog2.q.out ql/src/test/results/clientpositive/constprog2.q.out index 245a51f..37ec13c 100644 --- ql/src/test/results/clientpositive/constprog2.q.out +++ ql/src/test/results/clientpositive/constprog2.q.out @@ -1,14 +1,8 @@ -PREHOOK: query: -- Some constant propagation does not work in CBO due to added casts(?), disable for now. - - -EXPLAIN +PREHOOK: query: EXPLAIN SELECT src1.key, src1.key + 1, src2.value FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 86 PREHOOK: type: QUERY -POSTHOOK: query: -- Some constant propagation does not work in CBO due to added casts(?), disable for now. - - -EXPLAIN +POSTHOOK: query: EXPLAIN SELECT src1.key, src1.key + 1, src2.value FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 86 POSTHOOK: type: QUERY @@ -21,37 +15,43 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: src2 + alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 86) (type: boolean) + predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '86' (type: string) - sort order: + + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: '86' (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 86) (type: boolean) + predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '86' (type: string) - sort order: + + Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '86' (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 - 1 {VALUE._col0} - outputColumnNames: _col6 + 0 {VALUE._col0} + 1 + outputColumnNames: _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '86' (type: string), 87.0 (type: double), _col6 (type: string) + expressions: '86' (type: string), 87.0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -96,37 +96,43 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: src2 + alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(key) = 86) (type: boolean) + predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '86' (type: string) - sort order: + + Select Operator + expressions: value (type: string) + outputColumnNames: _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: '86' (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (UDFToDouble(key) = 86) (type: boolean) + predicate: (UDFToDouble(key) = 86.0) (type: boolean) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: '86' (type: string) - sort order: + + Select Operator Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: '86' (type: string) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 condition expressions: - 0 - 1 {VALUE._col0} - outputColumnNames: _col6 + 0 {VALUE._col0} + 1 + outputColumnNames: _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: '86' (type: string), 87.0 (type: double), _col6 (type: string) + expressions: '86' (type: string), 87.0 (type: double), _col1 (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/correlationoptimizer8.q.out ql/src/test/results/clientpositive/correlationoptimizer8.q.out index 3c737d5..5a8827c 100644 --- ql/src/test/results/clientpositive/correlationoptimizer8.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer8.q.out @@ -34,8 +34,8 @@ JOIN src1 x ON (x.key = subq1.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-4 is a root stage Stage-0 depends on stages: Stage-2 STAGE PLANS: @@ -46,15 +46,15 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) + predicate: (UDFToDouble(key) < 20.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -88,33 +88,45 @@ STAGE PLANS: TableScan Union Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + Union + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator condition map: @@ -136,22 +148,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan - alias: x1 + alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) + predicate: (UDFToDouble(key) > 100.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -250,15 +262,15 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) + predicate: (UDFToDouble(key) < 20.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -269,18 +281,18 @@ STAGE PLANS: Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan - alias: x1 + alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) + predicate: (UDFToDouble(key) > 100.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -294,85 +306,97 @@ STAGE PLANS: alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Demux Operator - Statistics: Num rows: 345 Data size: 3625 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 340 Data size: 3587 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 170 Data size: 1793 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 170 Data size: 1793 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 344 Data size: 3614 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 689 Data size: 7239 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 340 Data size: 3586 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 340 Data size: 3586 Basic stats: COMPLETE Column stats: NONE + Mux Operator + Statistics: Num rows: 680 Data size: 7173 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 170 Data size: 1793 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 172 Data size: 1807 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 170 Data size: 1793 Basic stats: COMPLETE Column stats: NONE Union - Statistics: Num rows: 344 Data size: 3614 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 689 Data size: 7239 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Inner Join 0 to 1 - condition expressions: - 0 {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - outputColumnNames: _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) - outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 340 Data size: 3586 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 340 Data size: 3586 Basic stats: COMPLETE Column stats: NONE + Mux Operator + Statistics: Num rows: 680 Data size: 7173 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col2 (type: string), _col3 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator - Statistics: Num rows: 689 Data size: 7239 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 680 Data size: 7173 Basic stats: COMPLETE Column stats: NONE Join Operator condition map: Inner Join 0 to 1 @@ -475,26 +499,26 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: x1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) > 100.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + expressions: value (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -502,11 +526,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -520,30 +544,42 @@ STAGE PLANS: TableScan Union Statistics: Num rows: 87 Data size: 911 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 87 Data size: 911 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 87 Data size: 911 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan Union Statistics: Num rows: 87 Data size: 911 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 87 Data size: 911 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 87 Data size: 911 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan - alias: x + alias: x1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) Reduce Operator Tree: Join Operator condition map: @@ -569,26 +605,26 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: x1 - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) < 20.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: value (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -596,11 +632,11 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false table: @@ -696,58 +732,62 @@ STAGE PLANS: Map Reduce Map Operator Tree: TableScan - alias: x - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + alias: x1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) > 100.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + expressions: value (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) TableScan alias: x1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + predicate: (UDFToDouble(key) < 20.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string) - outputColumnNames: value - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: value (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) Reduce Operator Tree: Demux Operator Statistics: Num rows: 199 Data size: 2015 Basic stats: COMPLETE Column stats: NONE @@ -763,27 +803,31 @@ STAGE PLANS: Statistics: Num rows: 99 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 198 Data size: 2004 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 397 Data size: 4019 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 198 Data size: 2004 Basic stats: COMPLETE Column stats: NONE + Mux Operator + Statistics: Num rows: 397 Data size: 4019 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Group By Operator aggregations: count(VALUE._col0) keys: KEY._col0 (type: string) @@ -796,27 +840,31 @@ STAGE PLANS: Statistics: Num rows: 99 Data size: 1002 Basic stats: COMPLETE Column stats: NONE Union Statistics: Num rows: 198 Data size: 2004 Basic stats: COMPLETE Column stats: NONE - Mux Operator - Statistics: Num rows: 397 Data size: 4019 Basic stats: COMPLETE Column stats: NONE - Join Operator - condition map: - Left Outer Join0 to 1 - condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 198 Data size: 2004 Basic stats: COMPLETE Column stats: NONE + Mux Operator + Statistics: Num rows: 397 Data size: 4019 Basic stats: COMPLETE Column stats: NONE + Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - File Output Operator - compressed: false + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Mux Operator Statistics: Num rows: 397 Data size: 4019 Basic stats: COMPLETE Column stats: NONE Join Operator @@ -926,8 +974,8 @@ JOIN src1 x ON (x.key = subq1.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-4 is a root stage Stage-0 depends on stages: Stage-2 STAGE PLANS: @@ -938,15 +986,15 @@ STAGE PLANS: alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) + predicate: (UDFToDouble(key) < 20.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -980,33 +1028,45 @@ STAGE PLANS: TableScan Union Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - TableScan - Union - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) TableScan alias: x Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: key is not null (type: boolean) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: (((UDFToDouble(key) < 20.0) or (UDFToDouble(key) > 100.0)) and key is not null) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + TableScan + Union + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) Reduce Operator Tree: Join Operator condition map: @@ -1028,22 +1088,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan - alias: x1 + alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) + predicate: (UDFToDouble(key) > 100.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: key, value + expressions: key (type: string), value (type: string), 1 (type: int) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string), value (type: string) + aggregations: count(_col2) + keys: _col0 (type: string), _col1 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -1077,6 +1137,7 @@ STAGE PLANS: Processor Tree: ListSink +WARNING: Comparing a bigint and a string may result in a loss of precision. PREHOOK: query: -- When the Correlation Optimizer is turned on, because a input of UnionOperator is -- not correlated, we cannot handle this case right now. So, this query will not be -- optimized. @@ -1102,28 +1163,28 @@ FROM FULL OUTER JOIN src1 x ON (x.key = subq1.key) POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-3 - Stage-3 is a root stage - Stage-0 depends on stages: Stage-2 + Stage-2 is a root stage + Stage-1 depends on stages: Stage-2, Stage-4 + Stage-4 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-2 Map Reduce Map Operator Tree: TableScan alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key < 20) (type: boolean) + predicate: (UDFToDouble(key) < 20.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE @@ -1151,36 +1212,48 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - Stage: Stage-2 + Stage: Stage-1 Map Reduce Map Operator Tree: TableScan - Union - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToDouble(_col0) (type: double) sort order: + Map-reduce partition columns: UDFToDouble(_col0) (type: double) - Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) TableScan Union Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(_col0) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint), _col1 (type: bigint) + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) TableScan - alias: x - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: UDFToDouble(key) (type: double) - sort order: + - Map-reduce partition columns: UDFToDouble(key) (type: double) - Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: key (type: string), value (type: string) + Union + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), _col1 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 166 Data size: 1762 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint), _col1 (type: bigint) Reduce Operator Tree: Join Operator condition map: @@ -1202,22 +1275,22 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-3 + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan - alias: x1 + alias: x Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key > 100) (type: boolean) + predicate: (UDFToDouble(key) > 100.0) (type: boolean) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string) - outputColumnNames: key + expressions: key (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: key (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE diff --git ql/src/test/results/clientpositive/filter_join_breaktask2.q.out ql/src/test/results/clientpositive/filter_join_breaktask2.q.out index d81bc93..6a2396e 100644 --- ql/src/test/results/clientpositive/filter_join_breaktask2.q.out +++ ql/src/test/results/clientpositive/filter_join_breaktask2.q.out @@ -230,6 +230,7 @@ POSTHOOK: Input: default@t4 POSTHOOK: Input: default@t4@ds=2010-04-17 #### A masked pattern was here #### 4 1 1 8 4 5 1 0 9 U 2 2 0 2 1 1 J C A U 2 s 2 NULL NULL NULL NULL NULL NULL 1 j S 6 NULL 1 2 J g 1 e 2 1 2 U P p 3 0 0 0 1 1 1 0 0 0 6 2 j NULL NULL NULL NULL NULL NULL 5 NULL NULL j 2 2 1 2 2 1 1 1 1 1 1 1 1 32 NULL 2010-04-17 +WARNING: Comparing a bigint and a string may result in a loss of precision. PREHOOK: query: SELECT a.c1 as a_c1, b.c1 b_c1, d.c0 as d_c0 FROM T1 a JOIN T2 b ON (a.c1 = b.c1 AND a.ds='2010-04-17' AND b.ds='2010-04-17') diff --git ql/src/test/results/clientpositive/infer_bucket_sort.q.out ql/src/test/results/clientpositive/infer_bucket_sort.q.out index a2252df..8d3080b 100644 --- ql/src/test/results/clientpositive/infer_bucket_sort.q.out +++ ql/src/test/results/clientpositive/infer_bucket_sort.q.out @@ -1,6 +1,5 @@ PREHOOK: query: -- This tests inferring how data is bucketed/sorted from the operators in the reducer -- and populating that information in partitions' metadata --- CBO is disabled; it rewrites query plans, messing with the specifically crafted test cases. CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING) PREHOOK: type: CREATETABLE @@ -8,7 +7,6 @@ PREHOOK: Output: database:default PREHOOK: Output: default@test_table POSTHOOK: query: -- This tests inferring how data is bucketed/sorted from the operators in the reducer -- and populating that information in partitions' metadata --- CBO is disabled; it rewrites query plans, messing with the specifically crafted test cases. CREATE TABLE test_table (key STRING, value STRING) PARTITIONED BY (part STRING) POSTHOOK: type: CREATETABLE @@ -302,7 +300,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_table@part=1 POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)c.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)a.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1') PREHOOK: type: DESCTABLE PREHOOK: Input: default@test_table @@ -357,7 +355,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_table@part=1 POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)c.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table PARTITION(part=1).value EXPRESSION [(src)a.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1') PREHOOK: type: DESCTABLE PREHOOK: Input: default@test_table @@ -395,8 +393,8 @@ InputFormat: org.apache.hadoop.mapred.TextInputFormat OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Compressed: No Num Buckets: 1 -Bucket Columns: [value] -Sort Columns: [Order(col:value, order:1)] +Bucket Columns: [key] +Sort Columns: [Order(col:key, order:1)] Storage Desc Params: serialization.format 1 PREHOOK: query: -- Test distribute by, should only be bucketed by key @@ -687,7 +685,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_table@part=1 POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)b.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)a.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1') PREHOOK: type: DESCTABLE PREHOOK: Input: default@test_table @@ -742,7 +740,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@test_table@part=1 POSTHOOK: Lineage: test_table PARTITION(part=1).key SIMPLE [(src)a.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)b.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: test_table PARTITION(part=1).value SIMPLE [(src)a.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: DESCRIBE FORMATTED test_table PARTITION (part = '1') PREHOOK: type: DESCTABLE PREHOOK: Input: default@test_table diff --git ql/src/test/results/clientpositive/join12.q.out ql/src/test/results/clientpositive/join12.q.out index 11a9d9c..d959989 100644 --- ql/src/test/results/clientpositive/join12.q.out +++ ql/src/test/results/clientpositive/join12.q.out @@ -1,6 +1,4 @@ -PREHOOK: query: -- Disable CBO for now as it eliminates the n-way join - -EXPLAIN +PREHOOK: query: EXPLAIN SELECT src1.c1, src2.c4 FROM (SELECT src.key as c1, src.value as c2 from src) src1 @@ -11,9 +9,7 @@ JOIN (SELECT src.key as c5, src.value as c6 from src) src3 ON src1.c1 = src3.c5 AND src3.c5 < 80 PREHOOK: type: QUERY -POSTHOOK: query: -- Disable CBO for now as it eliminates the n-way join - -EXPLAIN +POSTHOOK: query: EXPLAIN SELECT src1.c1, src2.c4 FROM (SELECT src.key as c1, src.value as c2 from src) src1 @@ -26,7 +22,8 @@ ON src1.c1 = src3.c5 AND src3.c5 < 80 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -36,23 +33,23 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((key < 100) and (key < 80)) and key is not null) (type: boolean) - Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 80) and (key < 100)) (type: boolean) + predicate: ((UDFToDouble(key) < 100.0) and (UDFToDouble(key) < 80.0)) (type: boolean) Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) @@ -63,39 +60,67 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {VALUE._col0} + 1 {KEY.reducesinkkey0} + outputColumnNames: _col1, _col2 + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((key < 100) and (key < 80)) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: (((UDFToDouble(key) < 80.0) and (UDFToDouble(key) < 100.0)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 condition expressions: - 0 {KEY.reducesinkkey0} - 1 {VALUE._col0} - 2 - outputColumnNames: _col0, _col3 - Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + 0 + 1 {KEY.reducesinkkey0} {VALUE._col1} + outputColumnNames: _col1, _col3 + Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: string), _col3 (type: string) + expressions: _col1 (type: string), _col3 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat diff --git ql/src/test/results/clientpositive/list_bucket_dml_1.q.out ql/src/test/results/clientpositive/list_bucket_dml_1.q.out index 735c200..a95d3fa 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_1.q.out @@ -207,8 +207,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -523,7 +523,7 @@ STAGE PLANS: name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part Truncated Path -> Alias: - /list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=484 [list_bucketing_dynamic_part] + /list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=484 [$hdt$_0:list_bucketing_dynamic_part] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_11.q.out ql/src/test/results/clientpositive/list_bucket_dml_11.q.out index 9c936a0..1d67f68 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_11.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_11.q.out @@ -393,7 +393,7 @@ STAGE PLANS: name: default.list_bucketing_static_part name: default.list_bucketing_static_part Truncated Path -> Alias: - /list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466 [list_bucketing_static_part] + /list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466 [$hdt$_0:list_bucketing_static_part] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_12.q.out ql/src/test/results/clientpositive/list_bucket_dml_12.q.out index ea4b6b8..a35c55d 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_12.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_12.q.out @@ -164,7 +164,7 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [src] + /src [$hdt$_0:src] Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_13.q.out ql/src/test/results/clientpositive/list_bucket_dml_13.q.out index 9b30a84..6230968 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_13.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_13.q.out @@ -164,7 +164,7 @@ STAGE PLANS: name: default.src name: default.src Truncated Path -> Alias: - /src [src] + /src [$hdt$_0:src] Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_2.q.out ql/src/test/results/clientpositive/list_bucket_dml_2.q.out index 01f7218..19cee53 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_2.q.out @@ -229,8 +229,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_3.q.out ql/src/test/results/clientpositive/list_bucket_dml_3.q.out index 227b55e..f068b3d 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_3.q.out @@ -199,8 +199,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -463,7 +463,7 @@ STAGE PLANS: name: default.list_bucketing_static_part name: default.list_bucketing_static_part Truncated Path -> Alias: - /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484 [list_bucketing_static_part] + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484 [$hdt$_0:list_bucketing_static_part] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_4.q.out ql/src/test/results/clientpositive/list_bucket_dml_4.q.out index b517c83..5216a8a 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_4.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_4.q.out @@ -237,8 +237,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -532,8 +532,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-7 Conditional Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_5.q.out ql/src/test/results/clientpositive/list_bucket_dml_5.q.out index d8a0abf..2cd6dfe 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_5.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_5.q.out @@ -209,8 +209,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator diff --git ql/src/test/results/clientpositive/list_bucket_dml_6.q.out ql/src/test/results/clientpositive/list_bucket_dml_6.q.out index 37db2cf..2cd36f2 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_6.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_6.q.out @@ -165,7 +165,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -293,8 +293,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -520,7 +520,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -648,8 +648,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-7 Conditional Operator @@ -975,52 +975,14 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: list_bucketing_dynamic_part - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((key = '484') and (value = 'val_484')) (type: boolean) - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: hr=a1 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -1064,9 +1026,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Partition - base file name: value=val_484 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -1110,15 +1070,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part - Truncated Path -> Alias: - /list_bucketing_dynamic_part/ds=2008-04-08/hr=a1 [list_bucketing_dynamic_part] - /list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484 [list_bucketing_dynamic_part] - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: list_bucketing_dynamic_part + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + ListSink PREHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/list_bucket_dml_7.q.out ql/src/test/results/clientpositive/list_bucket_dml_7.q.out index adaaba4..1c7d56f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_7.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_7.q.out @@ -113,7 +113,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -241,8 +241,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -468,7 +468,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -596,8 +596,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-7 Conditional Operator @@ -923,52 +923,14 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: list_bucketing_dynamic_part - Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((key = '484') and (value = 'val_484')) (type: boolean) - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: hr=a1 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -1012,9 +974,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Partition - base file name: hr=b1 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -1058,15 +1018,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part - Truncated Path -> Alias: - /list_bucketing_dynamic_part/ds=2008-04-08/hr=a1 [list_bucketing_dynamic_part] - /list_bucketing_dynamic_part/ds=2008-04-08/hr=b1 [list_bucketing_dynamic_part] - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: list_bucketing_dynamic_part + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE + ListSink PREHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/list_bucket_dml_8.q.out ql/src/test/results/clientpositive/list_bucket_dml_8.q.out index 297029a..ce6255f 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_8.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_8.q.out @@ -167,7 +167,7 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - expressions: key (type: string), value (type: string), if(((key % 100) = 0), 'a1', 'b1') (type: string) + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator @@ -295,8 +295,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -566,52 +566,14 @@ TOK_QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: list_bucketing_dynamic_part - Statistics: Num rows: 16 Data size: 136 Basic stats: PARTIAL Column stats: NONE - GatherStats: false - Filter Operator - isSamplingPred: false - predicate: ((key = '484') and (value = 'val_484')) (type: boolean) - Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 0 -#### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE -#### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - columns _col0,_col1,_col2,_col3 - columns.types string:string:string:string - escape.delim \ - hive.serialization.extend.nesting.levels true - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - TotalFiles: 1 - GatherStats: false - MultiFileSpray: false - Path -> Alias: -#### A masked pattern was here #### - Path -> Partition: -#### A masked pattern was here #### + Stage: Stage-0 + Fetch Operator + limit: -1 + Partition Description: Partition - base file name: hr=a1 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -655,9 +617,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part -#### A masked pattern was here #### Partition - base file name: hr=b1 input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat partition values: @@ -701,15 +661,20 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.list_bucketing_dynamic_part name: default.list_bucketing_dynamic_part - Truncated Path -> Alias: - /list_bucketing_dynamic_part/ds=2008-04-08/hr=a1 [list_bucketing_dynamic_part] - /list_bucketing_dynamic_part/ds=2008-04-08/hr=b1 [list_bucketing_dynamic_part] - - Stage: Stage-0 - Fetch Operator - limit: -1 Processor Tree: - ListSink + TableScan + alias: list_bucketing_dynamic_part + Statistics: Num rows: 16 Data size: 136 Basic stats: PARTIAL Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((key = '484') and (value = 'val_484')) (type: boolean) + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE + ListSink PREHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484' PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/list_bucket_dml_9.q.out ql/src/test/results/clientpositive/list_bucket_dml_9.q.out index 135cd78..d4f42ca 100644 --- ql/src/test/results/clientpositive/list_bucket_dml_9.q.out +++ ql/src/test/results/clientpositive/list_bucket_dml_9.q.out @@ -237,8 +237,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-0 Move Operator @@ -532,8 +532,8 @@ STAGE PLANS: name: default.srcpart name: default.srcpart Truncated Path -> Alias: - /srcpart/ds=2008-04-08/hr=11 [srcpart] - /srcpart/ds=2008-04-08/hr=12 [srcpart] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart] + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart] Stage: Stage-7 Conditional Operator diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out index 5e9c0ff..6def2f2 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out @@ -246,7 +246,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/key=484/value=val_484 [fact_daily] + /fact_daily/ds=1/hr=4/key=484/value=val_484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -406,7 +406,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/key=238/value=val_238 [fact_daily] + /fact_daily/ds=1/hr=4/key=238/value=val_238 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -559,7 +559,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] + /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -713,7 +713,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] + /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out index 2fc5d7f..fbf1bb0 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out @@ -290,8 +290,8 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] - /fact_daily/ds=1/hr=4/key=484/value=val_484 [fact_daily] + /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] + /fact_daily/ds=1/hr=4/key=484/value=val_484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -443,7 +443,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] + /fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -662,8 +662,8 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=4/key=238/value=val_238 [fact_daily] - /fact_daily/ds=1/hr=4/key=484/value=val_484 [fact_daily] + /fact_daily/ds=1/hr=4/key=238/value=val_238 [$hdt$_0:fact_daily] + /fact_daily/ds=1/hr=4/key=484/value=val_484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out index d738dbf..d7bace4 100644 --- ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out @@ -489,7 +489,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_daily/ds=1/hr=1 [fact_daily] + /fact_daily/ds=1/hr=1 [$hdt$_0:$hdt$_0:fact_daily] Needs Tagging: false Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out index f0c681c..01f673a 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out @@ -299,7 +299,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/x=484 [fact_daily] + /fact_tz/ds=1/x=484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -446,7 +446,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] + /fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -589,7 +589,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] + /fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out index 0ef749a..b6fc8b5 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out @@ -316,7 +316,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/x=484 [subq:fact_daily] + /fact_tz/ds=1/x=484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -483,7 +483,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/x=484 [subq:fact_daily] + /fact_tz/ds=1/x=484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator @@ -563,12 +563,12 @@ STAGE PLANS: predicate: (x = 484) (type: boolean) Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - expressions: y (type: string) - outputColumnNames: y + expressions: y (type: string), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator - aggregations: count(1) - keys: y (type: string) + aggregations: count(_col1) + keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE @@ -630,7 +630,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/x=484 [fact_daily] + /fact_tz/ds=1/x=484 [$hdt$_0:$hdt$_0:fact_daily] Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -759,12 +759,12 @@ STAGE PLANS: predicate: (x = 484) (type: boolean) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 484 (type: int) - outputColumnNames: x + expressions: 484 (type: int), 1 (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Group By Operator - aggregations: count(1) - keys: x (type: int) + aggregations: count(_col1) + keys: _col0 (type: int) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE @@ -826,7 +826,7 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/x=484 [subq:fact_daily] + /fact_tz/ds=1/x=484 [$hdt$_0:$hdt$_0:fact_daily] Needs Tagging: false Reduce Operator Tree: Group By Operator @@ -836,7 +836,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: 484 (type: int), _col1 (type: bigint) + expressions: _col0 (type: int), _col1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out index d6a075b..4de1fdc 100644 --- ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out +++ ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out @@ -419,9 +419,9 @@ STAGE PLANS: name: default.fact_daily name: default.fact_daily Truncated Path -> Alias: - /fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [fact_daily] - /fact_tz/ds=1/x=238 [fact_daily] - /fact_tz/ds=1/x=484 [fact_daily] + /fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [$hdt$_0:fact_daily] + /fact_tz/ds=1/x=238 [$hdt$_0:fact_daily] + /fact_tz/ds=1/x=484 [$hdt$_0:fact_daily] Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/partition_wise_fileformat2.q.out ql/src/test/results/clientpositive/partition_wise_fileformat2.q.out index 99d944b..c8abc9e 100644 --- ql/src/test/results/clientpositive/partition_wise_fileformat2.q.out +++ ql/src/test/results/clientpositive/partition_wise_fileformat2.q.out @@ -1,14 +1,10 @@ -PREHOOK: query: -- Disable CBO since it causes filter pruning to fail to prune in this case - --- SORT_BEFORE_DIFF +PREHOOK: query: -- SORT_BEFORE_DIFF create table partition_test_partitioned(key string, value string) partitioned by (dt string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@partition_test_partitioned -POSTHOOK: query: -- Disable CBO since it causes filter pruning to fail to prune in this case - --- SORT_BEFORE_DIFF +POSTHOOK: query: -- SORT_BEFORE_DIFF create table partition_test_partitioned(key string, value string) partitioned by (dt string) POSTHOOK: type: CREATETABLE @@ -75,11 +71,14 @@ STAGE PLANS: TableScan alias: partition_test_partitioned Statistics: Num rows: 75 Data size: 548 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), dt (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 75 Data size: 548 Basic stats: COMPLETE Column stats: NONE - ListSink + Filter Operator + predicate: ((UDFToDouble(dt) >= 100.0) and (UDFToDouble(dt) <= 102.0)) (type: boolean) + Statistics: Num rows: 8 Data size: 58 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), dt (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 58 Basic stats: COMPLETE Column stats: NONE + ListSink PREHOOK: query: select * from partition_test_partitioned where dt >=100 and dt <= 102 PREHOOK: type: QUERY @@ -185,11 +184,14 @@ STAGE PLANS: TableScan alias: partition_test_partitioned Statistics: Num rows: 75 Data size: 548 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: key (type: string), value (type: string), dt (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 75 Data size: 548 Basic stats: COMPLETE Column stats: NONE - ListSink + Filter Operator + predicate: ((UDFToDouble(dt) >= 100.0) and (UDFToDouble(dt) <= 102.0)) (type: boolean) + Statistics: Num rows: 8 Data size: 58 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), dt (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 8 Data size: 58 Basic stats: COMPLETE Column stats: NONE + ListSink PREHOOK: query: select * from partition_test_partitioned where dt >=100 and dt <= 102 PREHOOK: type: QUERY diff --git ql/src/test/results/clientpositive/ppd_join3.q.out ql/src/test/results/clientpositive/ppd_join3.q.out index 2993372..6f39429 100644 --- ql/src/test/results/clientpositive/ppd_join3.q.out +++ ql/src/test/results/clientpositive/ppd_join3.q.out @@ -1,6 +1,4 @@ -PREHOOK: query: -- Disable CBO for now as it eliminates the n-way join - -EXPLAIN +PREHOOK: query: EXPLAIN SELECT src1.c1, src2.c4 FROM (SELECT src.key as c1, src.value as c2 from src where src.key <> '11' ) src1 @@ -12,9 +10,7 @@ JOIN ON src1.c1 = src3.c5 WHERE src1.c1 > '0' and (src1.c2 <> 'val_500' or src1.c1 > '1') and (src2.c3 > '10' or src1.c1 <> '10') and (src2.c3 <> '4') and (src3.c5 <> '1') PREHOOK: type: QUERY -POSTHOOK: query: -- Disable CBO for now as it eliminates the n-way join - -EXPLAIN +POSTHOOK: query: EXPLAIN SELECT src1.c1, src2.c4 FROM (SELECT src.key as c1, src.value as c2 from src where src.key <> '11' ) src1 @@ -28,7 +24,8 @@ WHERE src1.c1 > '0' and (src1.c2 <> 'val_500' or src1.c1 > '1') and (src2.c3 > ' POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -38,83 +35,145 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((((key <> '12') and (key <> '4')) and (key <> '11')) and (key > '0')) and (key < '400')) and key is not null) and (key <> '13')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + predicate: ((_col0 <> '11') and ((_col0 > '0') and (_col0 < '400'))) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE + predicate: (((((((((key <> '11') and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key < '400')) and (key <> '12')) and (key <> '4')) and key is not null) and (key <> '13')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + predicate: ((_col0 <> '12') and (_col0 <> '4')) (type: boolean) + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 19 Data size: 201 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 19 Data size: 201 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col1 > '10') or (_col0 <> '10')) (type: boolean) + Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 <> '13') and (_col0 <> '1')) (type: boolean) + Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 20 Data size: 221 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 110 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((((key <> '13') and (key <> '1')) and (key <> '11')) and (key > '0')) and (key < '400')) and (key <> '12')) and (key <> '4')) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (_col0 < '400') (type: boolean) - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE + predicate: ((_col0 <> '11') and ((_col0 > '0') and ((_col0 < '400') and ((_col0 <> '12') and (_col0 <> '4'))))) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - 2 {KEY.reducesinkkey0} - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 52 Data size: 558 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 + 1 {KEY.reducesinkkey0} {VALUE._col1} + outputColumnNames: _col1, _col3 + Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1749,7 +1808,8 @@ WHERE src1.c1 > '0' and (src1.c2 <> 'val_500' or src1.c1 > '1') and (src2.c3 > ' POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -1759,74 +1819,101 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((((key <> '12') and (key <> '4')) and (key <> '11')) and (key > '0')) and (key < '400')) and key is not null) and (key <> '13')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean) - Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE + predicate: (((((((((key <> '11') and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key < '400')) and (key <> '12')) and (key <> '4')) and key is not null) and (key <> '13')) and (key <> '1')) (type: boolean) + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string) + Statistics: Num rows: 37 Data size: 393 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 40 Data size: 432 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 > '10') or (_col2 <> '10')) (type: boolean) + Statistics: Num rows: 40 Data size: 432 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 40 Data size: 432 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 40 Data size: 432 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: string) TableScan alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((((key <> '13') and (key <> '1')) and (key <> '11')) and (key > '0')) and (key < '400')) and (key <> '12')) and (key <> '4')) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: string) outputColumnNames: _col0 - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 - Inner Join 0 to 2 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - 2 {KEY.reducesinkkey0} - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 160 Data size: 1705 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean) - Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col3 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 + 1 {KEY.reducesinkkey0} {VALUE._col1} + outputColumnNames: _col1, _col3 + Statistics: Num rows: 44 Data size: 475 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 44 Data size: 475 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 44 Data size: 475 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git ql/src/test/results/clientpositive/ppd_outer_join4.q.out ql/src/test/results/clientpositive/ppd_outer_join4.q.out index 31b52f5..6b9afb7 100644 --- ql/src/test/results/clientpositive/ppd_outer_join4.q.out +++ ql/src/test/results/clientpositive/ppd_outer_join4.q.out @@ -1,6 +1,4 @@ -PREHOOK: query: -- Disable CBO for now as it eliminates the n-way join - -EXPLAIN +PREHOOK: query: EXPLAIN FROM src a LEFT OUTER JOIN @@ -12,9 +10,7 @@ EXPLAIN SELECT a.key, a.value, b.key, b.value, c.key WHERE a.key > '10' AND a.key < '20' AND b.key > '15' AND b.key < '25' AND sqrt(c.key) <> 13 PREHOOK: type: QUERY -POSTHOOK: query: -- Disable CBO for now as it eliminates the n-way join - -EXPLAIN +POSTHOOK: query: EXPLAIN FROM src a LEFT OUTER JOIN @@ -28,72 +24,150 @@ EXPLAIN POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 Map Reduce Map Operator Tree: TableScan - alias: b + alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sqrt(key) <> 13) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: ((((((key > '15') and (key < '25')) and (key > '10')) and (key < '20')) and key is not null) and (sqrt(key) <> 13.0)) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 > '10') and (_col0 < '20')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string) TableScan - alias: c + alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sqrt(key) <> 13) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key > '10') and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) and (sqrt(key) <> 13.0)) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 > '15') and (_col0 < '25')) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: (sqrt(_col0) <> 13.0) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sqrt(key) <> 13) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: ((((((sqrt(key) <> 13.0) and (key > '10')) and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((_col0 > '10') and ((_col0 < '20') and ((_col0 > '15') and (_col0 < '25')))) (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: _col0 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Reduce Operator Tree: Join Operator condition map: - Left Outer Join0 to 1 - Right Outer Join0 to 2 + Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - 2 {KEY.reducesinkkey0} - outputColumnNames: _col0, _col1, _col5, _col6, _col10 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (((((_col0 > '10') and (_col0 < '20')) and (_col5 > '15')) and (_col5 < '25')) and (sqrt(_col10) <> 13)) (type: boolean) - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 {KEY.reducesinkkey0} + 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -383,72 +457,110 @@ POSTHOOK: query: EXPLAIN POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 Map Reduce Map Operator Tree: TableScan - alias: b + alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sqrt(key) <> 13) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: ((((((key > '15') and (key < '25')) and (key > '10')) and (key < '20')) and key is not null) and (sqrt(key) <> 13.0)) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) TableScan - alias: c + alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sqrt(key) <> 13) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + predicate: ((((((key > '10') and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) and (sqrt(key) <> 13.0)) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string) TableScan alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (sqrt(key) <> 13) (type: boolean) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: key (type: string) - sort order: + - Map-reduce partition columns: key (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: value (type: string) + predicate: ((((((sqrt(key) <> 13.0) and (key > '10')) and (key < '20')) and (key > '15')) and (key < '25')) and key is not null) (type: boolean) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 3 Data size: 31 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Join Operator condition map: - Left Outer Join0 to 1 - Right Outer Join0 to 2 + Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {KEY.reducesinkkey0} {VALUE._col0} - 2 {KEY.reducesinkkey0} - outputColumnNames: _col0, _col1, _col5, _col6, _col10 - Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((((_col5 > '15') and (_col5 < '25')) and (_col0 > '10')) and (_col0 < '20')) (type: boolean) - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 13 Data size: 138 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + 0 {KEY.reducesinkkey0} + 1 {KEY.reducesinkkey0} {VALUE._col0} {VALUE._col1} {VALUE._col2} + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 34 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator