From d2e5a25d86d44c6bc27c626c44ed5381423cc6cd Mon Sep 17 00:00:00 2001 From: Ashutosh Chauhan Date: Fri, 17 Nov 2017 17:28:15 -0800 Subject: [PATCH] HIVE-18100 : Some tests time out --- .../test/resources/testconfiguration.properties | 7 +- .../test/queries/clientpositive/explainanalyze_1.q | 8 + .../test/queries/clientpositive/explainanalyze_2.q | 8 - .../test/queries/clientpositive/unionDistinct_1.q | 315 +-- .../test/queries/clientpositive/unionDistinct_2.q | 23 - .../test/queries/clientpositive/unionDistinct_3.q | 182 ++ .../{tez => llap}/explainanalyze_2.q.out | 326 +-- .../clientpositive/llap/unionDistinct_2.q.out | 108 - .../clientpositive/llap/unionDistinct_3.q.out | 2650 ++++++++++++++++++++ .../clientpositive/tez/explainanalyze_1.q.out | 58 + .../clientpositive/tez/unionDistinct_2.q.out | 553 ---- 11 files changed, 3039 insertions(+), 1199 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/unionDistinct_3.q rename ql/src/test/results/clientpositive/{tez => llap}/explainanalyze_2.q.out (93%) create mode 100644 ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out delete mode 100644 ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 3b1005f85c..f78db6193b 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -43,7 +43,6 @@ disabled.query.files=ql_rewrite_gbtoidx.q,\ minitez.query.files.shared=delete_orig_table.q,\ orc_merge12.q,\ orc_vectorization_ppd.q,\ - unionDistinct_2.q,\ update_orig_table.q,\ vector_join_part_col_char.q,\ vector_non_string_partition.q,\ @@ -55,7 +54,6 @@ minitez.query.files.shared=delete_orig_table.q,\ minitez.query.files=acid_vectorization_original_tez.q,\ explainuser_3.q,\ explainanalyze_1.q,\ - explainanalyze_2.q,\ explainanalyze_3.q,\ explainanalyze_4.q,\ explainanalyze_5.q,\ @@ -80,7 +78,6 @@ minillap.shared.query.files=insert_into1.q,\ orc_merge_diff_fs.q,\ parallel_colstats.q,\ parquet_types_vectorization.q,\ - unionDistinct_1.q,\ union_type_chk.q,\ cte_2.q,\ cte_4.q,\ @@ -458,7 +455,6 @@ minillap.query.files=acid_bucket_pruning.q,\ temp_table_external.q,\ table_nonprintable.q,\ llap_nullscan.q,\ - unionDistinct_1.q,\ rcfile_merge2.q,\ rcfile_merge3.q,\ rcfile_merge4.q,\ @@ -531,6 +527,7 @@ minillaplocal.query.files=\ escape1.q,\ escape2.q,\ exchgpartition2lel.q,\ + explainanalyze_2.q,\ explainuser_1.q,\ explainuser_4.q,\ groupby2.q,\ @@ -661,6 +658,7 @@ minillaplocal.query.files=\ union_fast_stats.q,\ union_remove_26.q,\ union_top_level.q,\ + unionDistinct_1.q,\ vector_auto_smb_mapjoin_14.q,\ vector_complex_all.q,\ vector_decimal_2.q,\ @@ -792,6 +790,7 @@ minillaplocal.query.files=\ smb_mapjoin_17.q,\ groupby_resolution.q,\ windowing_windowspec2.q,\ + unionDistinct_3.q,\ vectorized_join46.q,\ vectorized_multi_output_select.q,\ partialdhj.q diff --git a/ql/src/test/queries/clientpositive/explainanalyze_1.q b/ql/src/test/queries/clientpositive/explainanalyze_1.q index 0a33a4f89c..6611922016 100644 --- a/ql/src/test/queries/clientpositive/explainanalyze_1.q +++ b/ql/src/test/queries/clientpositive/explainanalyze_1.q @@ -37,3 +37,11 @@ set hive.auto.convert.join.noconditionaltask.size=10000; EXPLAIN analyze SELECT x.key, y.value FROM src x JOIN src y ON (x.key = y.key); + +set hive.entity.capture.transform=true; +explain analyze +SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key; + diff --git a/ql/src/test/queries/clientpositive/explainanalyze_2.q b/ql/src/test/queries/clientpositive/explainanalyze_2.q index b764d7d2af..dd515ec5cc 100644 --- a/ql/src/test/queries/clientpositive/explainanalyze_2.q +++ b/ql/src/test/queries/clientpositive/explainanalyze_2.q @@ -1,7 +1,6 @@ set hive.map.aggr=false; set hive.strict.checks.bucketing=false; - set hive.explain.user=true; explain analyze @@ -139,13 +138,6 @@ INSERT OVERWRITE TABLE DEST2 SELECT unionsrc.key, unionsrc.value, COUNT(DISTINCT explain analyze FROM UNIQUEJOIN PRESERVE src a (a.key), PRESERVE src1 b (b.key), PRESERVE srcpart c (c.key) SELECT a.key, b.key, c.key; -set hive.entity.capture.transform=true; - -explain analyze -SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key; explain analyze FROM ( diff --git a/ql/src/test/queries/clientpositive/unionDistinct_1.q b/ql/src/test/queries/clientpositive/unionDistinct_1.q index f2a0dc17d1..4978b563af 100644 --- a/ql/src/test/queries/clientpositive/unionDistinct_1.q +++ b/ql/src/test/queries/clientpositive/unionDistinct_1.q @@ -1,3 +1,4 @@ +set hive.llap.execution.mode=auto; set hive.mapred.mode=nonstrict; set hive.explain.user=false; -- SORT_QUERY_RESULTS @@ -27,27 +28,6 @@ insert overwrite table tmptable select * from tmptable x sort by x.key; --- union11.q - - --- SORT_BEFORE_DIFF --- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; - - - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2 - UNION DISTINCT - select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; - - -- union12.q @@ -85,110 +65,6 @@ explain select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT select s2.key as key, s2.value as value from src s2) unionsrc; --- union14.q - - --- SORT_BEFORE_DIFF --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key; - - - - select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select 'tst1' as key, cast(count(1) as string) as value from src s1) - unionsrc group by unionsrc.key; --- union15.q - - --- SORT_BEFORE_DIFF --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; - - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2 - UNION DISTINCT - select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; - - --- union16.q - --- SORT_BEFORE_DIFF -EXPLAIN -SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src; - - -SELECT count(1) FROM ( - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src UNION DISTINCT - SELECT key, value FROM src) src; -- union17.q CREATE TABLE DEST1(key STRING, value STRING) STORED AS TEXTFILE; @@ -261,76 +137,6 @@ SELECT DEST119.* FROM DEST119 SORT BY DEST119.key, DEST119.value; SELECT DEST219.* FROM DEST219 SORT BY DEST219.key, DEST219.val1, DEST219.val2; - --- union2.q - --- SORT_BEFORE_DIFF --- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink - -explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc; - -select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2) unionsrc; --- union20.q - --- SORT_BEFORE_DIFF --- union :map-reduce sub-queries followed by join - -explain -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key); - -SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value -FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 -JOIN - (select 'tst1' as key, cast(count(1) as string) as value from src s3 - UNION DISTINCT - select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 -ON (unionsrc1.key = unionsrc2.key); --- union21.q - --- SORT_BEFORE_DIFF --- union of constants, udf outputs, and columns from text table and thrift table - -explain -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key; - -SELECT key, count(1) -FROM ( - SELECT '1' as key from src - UNION DISTINCT - SELECT reverse(key) as key from src - UNION DISTINCT - SELECT key as key from src - UNION DISTINCT - SELECT astring as key from src_thrift - UNION DISTINCT - SELECT lstring[0] as key from src_thrift -) union_output -GROUP BY key; -- union22.q -- SORT_QUERY_RESULTS @@ -819,11 +625,8 @@ insert overwrite table t6 select * from t5; select * from t6; -drop table t1; -drop table t2; -create table t1 as select * from src where key < 10; -create table t2 as select key, count(1) as cnt from src where key < 10 group by key; +create table t9 as select key, count(1) as cnt from src where key < 10 group by key; create table t7(c1 string, cnt int); create table t8(c1 string, cnt int); @@ -833,7 +636,7 @@ from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT - select key as c1, cnt from t2 + select key as c1, cnt from t9 ) x insert overwrite table t7 select c1, count(1) group by c1 @@ -844,7 +647,7 @@ from ( select key as c1, count(1) as cnt from t1 group by key UNION DISTINCT - select key as c1, cnt from t2 + select key as c1, cnt from t9 ) x insert overwrite table t7 select c1, count(1) group by c1 @@ -860,13 +663,6 @@ select * from t8; -- This tests various union queries which have columns on one side of the query -- being of double type and those on the other side another -drop table if exists t1; - -drop table if exists t2; - -CREATE TABLE t1 AS SELECT * FROM src WHERE key < 10; -CREATE TABLE t2 AS SELECT * FROM src WHERE key < 10; - -- Test simple union with double EXPLAIN SELECT * FROM @@ -1035,108 +831,5 @@ SELECT * FROM ( UNION DISTINCT SELECT key,value FROM (SELECT * FROM (SELECT * FROM src10_3) sub2 UNION DISTINCT SELECT * FROM src10_4 ) alias0 ) alias1; --- union4.q - - - --- union case: both subqueries are map-reduce jobs on same input, followed by filesink - -drop table if exists tmptable; - -create table tmptable(key string, value int); - -explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc; - -insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc; - -select * from tmptable x sort by x.key; - - --- union5.q - - --- SORT_BEFORE_DIFF --- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key; - -select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 - UNION DISTINCT - select 'tst2' as key, count(1) as value from src s2) unionsrc group by unionsrc.key; --- union6.q - - --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink - -drop table if exists tmptable; - -create table tmptable(key string, value string); - -explain -insert overwrite table tmptable - select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc; - -insert overwrite table tmptable -select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc; - -select * from tmptable x sort by x.key, x.value; - - --- union7.q - - - --- SORT_BEFORE_DIFF --- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink - -explain - select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key; - -select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 - UNION DISTINCT - select s2.key as key, s2.value as value from src1 s2) unionsrc group by unionsrc.key; - - --- union8.q - --- SORT_BEFORE_DIFF --- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink - -explain - select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; - -select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; --- union9.q - --- SORT_BEFORE_DIFF --- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by reducesink - -explain - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; - select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT - select s2.key as key, s2.value as value from src s2 UNION DISTINCT - select s3.key as key, s3.value as value from src s3) unionsrc; diff --git a/ql/src/test/queries/clientpositive/unionDistinct_2.q b/ql/src/test/queries/clientpositive/unionDistinct_2.q index 324bf68f58..da47d38b7b 100644 --- a/ql/src/test/queries/clientpositive/unionDistinct_2.q +++ b/ql/src/test/queries/clientpositive/unionDistinct_2.q @@ -6,21 +6,6 @@ CREATE TABLE u2 as select key, value from src order by key limit 3; CREATE TABLE u3 as select key, value from src order by key desc limit 5; -select * from u1; - -select * from u2; - -select * from u3; - -select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab; - select key, value from ( select key, value from u1 @@ -57,14 +42,6 @@ union select key as key, value from u3 ) tab; -select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab; select distinct * from ( diff --git a/ql/src/test/queries/clientpositive/unionDistinct_3.q b/ql/src/test/queries/clientpositive/unionDistinct_3.q new file mode 100644 index 0000000000..e3cf236928 --- /dev/null +++ b/ql/src/test/queries/clientpositive/unionDistinct_3.q @@ -0,0 +1,182 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; + +-- union2.q + +-- union case: both subqueries are map-reduce jobs on same input, followed by reduce sink + +explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; + +select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc; + +-- union6.q + + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by filesink + +drop table if exists tmptable; + +create table tmptable(key string, value string); + +explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc; + +select * from tmptable x sort by x.key, x.value; + +drop table if exists tmptable; + + +-- union8.q + +-- union case: all subqueries are a map-only jobs, 3 way union, same input for all sub-queries, followed by filesink + +explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; + +select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc; + +-- union11.q + + +-- union case: all subqueries are a map-reduce jobs, 3 way union, same input for all sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + + + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + +-- union14.q + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key; + + + + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key; +-- union15.q + + +-- union case: 1 subquery is a map-reduce job, different inputs for sub-queries, followed by reducesink + +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; + + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key; + +-- union16.q + +EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src; + + +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src; + +-- union20.q + +-- union :map-reduce sub-queries followed by join + +explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key); + +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key); +-- union21.q + +-- union of constants, udf outputs, and columns from text table and thrift table + +explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key; + +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key; + diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out similarity index 93% rename from ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out rename to ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out index e5b1d74654..7db3b13bbf 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out @@ -57,19 +57,19 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 7 + Reducer 7 llap File Output Operator [FS_56] Group By Operator [GBY_54] (rows=32/15 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] - <-Reducer 11 [CONTAINS] + <-Reducer 11 [CONTAINS] llap Reduce Output Operator [RS_53] PartitionCols:_col0, _col1 Select Operator [SEL_49] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_82] (rows=67/61 width=177) Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col1","_col2"] - <-Map 17 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_47] PartitionCols:_col0 Select Operator [SEL_42] (rows=500/500 width=87) @@ -78,12 +78,12 @@ Stage-0 predicate:key is not null TableScan [TS_40] (rows=500/500 width=87) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 10 [SIMPLE_EDGE] + <-Reducer 10 [SIMPLE_EDGE] llap SHUFFLE [RS_46] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_81] (rows=42/52 width=177) Conds:RS_43._col1=RS_44._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_44] PartitionCols:_col1 Select Operator [SEL_14] (rows=25/25 width=175) @@ -92,7 +92,7 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 15 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_43] PartitionCols:_col1 Select Operator [SEL_36] (rows=525/319 width=178) @@ -100,7 +100,7 @@ Stage-0 Group By Operator [GBY_35] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] + <-Map 13 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_27] (rows=25/25 width=175) @@ -109,7 +109,7 @@ Stage-0 predicate:value is not null TableScan [TS_25] (rows=25/25 width=175) Output:["key","value"] - <-Map 16 [CONTAINS] + <-Map 16 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_30] (rows=500/500 width=178) @@ -118,14 +118,14 @@ Stage-0 predicate:value is not null TableScan [TS_28] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] + <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_53] PartitionCols:_col0, _col1 Select Operator [SEL_24] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_80] (rows=67/61 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col2"] - <-Map 12 [SIMPLE_EDGE] + <-Map 12 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col0 Select Operator [SEL_17] (rows=500/500 width=87) @@ -134,16 +134,16 @@ Stage-0 predicate:key is not null TableScan [TS_15] (rows=500/500 width=87) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_79] (rows=42/52 width=177) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col1","_col2"] - <-Map 9 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 Select Operator [SEL_11] (rows=525/319 width=178) @@ -151,7 +151,7 @@ Stage-0 Group By Operator [GBY_10] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_2] (rows=25/25 width=175) @@ -160,7 +160,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25/25 width=175) Output:["key","value"] - <-Map 8 [CONTAINS] + <-Map 8 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_5] (rows=500/500 width=178) @@ -257,19 +257,19 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 9 + Reducer 9 llap File Output Operator [FS_114] Group By Operator [GBY_112] (rows=398/15 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 8 [SIMPLE_EDGE] - <-Reducer 15 [CONTAINS] + <-Reducer 15 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Select Operator [SEL_107] (rows=199/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_162] (rows=199/61 width=177) Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col2","_col5"] - <-Map 16 [SIMPLE_EDGE] + <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_105] PartitionCols:_col0 Select Operator [SEL_17] (rows=500/500 width=178) @@ -278,12 +278,12 @@ Stage-0 predicate:key is not null TableScan [TS_15] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 14 [SIMPLE_EDGE] + <-Reducer 14 [SIMPLE_EDGE] llap SHUFFLE [RS_104] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_161] (rows=123/52 width=86) Conds:RS_101._col1=RS_102._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_102] PartitionCols:_col1 Select Operator [SEL_14] (rows=25/25 width=175) @@ -292,7 +292,7 @@ Stage-0 predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 31 [SIMPLE_EDGE] + <-Reducer 31 [SIMPLE_EDGE] llap SHUFFLE [RS_101] PartitionCols:_col1 Select Operator [SEL_94] (rows=1525/319 width=178) @@ -300,7 +300,7 @@ Stage-0 Group By Operator [GBY_93] (rows=1525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 30 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] + <-Map 34 [CONTAINS] llap Reduce Output Operator [RS_92] PartitionCols:_col1, _col0 Select Operator [SEL_88] (rows=500/500 width=178) @@ -309,7 +309,7 @@ Stage-0 predicate:value is not null TableScan [TS_86] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 29 [CONTAINS] + <-Reducer 29 [CONTAINS] llap Reduce Output Operator [RS_92] PartitionCols:_col1, _col0 Select Operator [SEL_85] (rows=1025/319 width=178) @@ -317,7 +317,7 @@ Stage-0 Group By Operator [GBY_84] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 28 [SIMPLE_EDGE] - <-Map 33 [CONTAINS] + <-Map 33 [CONTAINS] llap Reduce Output Operator [RS_83] PartitionCols:_col1, _col0 Select Operator [SEL_79] (rows=500/500 width=178) @@ -326,7 +326,7 @@ Stage-0 predicate:value is not null TableScan [TS_77] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 27 [CONTAINS] + <-Reducer 27 [CONTAINS] llap Reduce Output Operator [RS_83] PartitionCols:_col1, _col0 Select Operator [SEL_76] (rows=525/319 width=178) @@ -334,7 +334,7 @@ Stage-0 Group By Operator [GBY_75] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 26 [SIMPLE_EDGE] - <-Map 25 [CONTAINS] + <-Map 25 [CONTAINS] llap Reduce Output Operator [RS_74] PartitionCols:_col1, _col0 Select Operator [SEL_67] (rows=25/25 width=175) @@ -343,7 +343,7 @@ Stage-0 predicate:value is not null TableScan [TS_65] (rows=25/25 width=175) Output:["key","value"] - <-Map 32 [CONTAINS] + <-Map 32 [CONTAINS] llap Reduce Output Operator [RS_74] PartitionCols:_col1, _col0 Select Operator [SEL_70] (rows=500/500 width=178) @@ -352,20 +352,20 @@ Stage-0 predicate:value is not null TableScan [TS_68] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 7 [CONTAINS] + <-Reducer 7 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Group By Operator [GBY_63] (rows=199/15 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] - <-Reducer 13 [CONTAINS] + <-Reducer 13 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_58] (rows=132/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_160] (rows=132/61 width=177) Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 24 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] llap SHUFFLE [RS_56] PartitionCols:_col0 Select Operator [SEL_51] (rows=500/500 width=178) @@ -374,16 +374,16 @@ Stage-0 predicate:key is not null TableScan [TS_49] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 12 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_159] (rows=82/52 width=86) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_53] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 21 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] llap SHUFFLE [RS_52] PartitionCols:_col1 Select Operator [SEL_45] (rows=1025/319 width=178) @@ -391,7 +391,7 @@ Stage-0 Group By Operator [GBY_44] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 20 [SIMPLE_EDGE] - <-Map 23 [CONTAINS] + <-Map 23 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_39] (rows=500/500 width=178) @@ -400,7 +400,7 @@ Stage-0 predicate:value is not null TableScan [TS_37] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 19 [CONTAINS] + <-Reducer 19 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_36] (rows=525/319 width=178) @@ -408,7 +408,7 @@ Stage-0 Group By Operator [GBY_35] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 18 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] + <-Map 17 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_27] (rows=25/25 width=175) @@ -417,7 +417,7 @@ Stage-0 predicate:value is not null TableScan [TS_25] (rows=25/25 width=175) Output:["key","value"] - <-Map 22 [CONTAINS] + <-Map 22 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_30] (rows=500/500 width=178) @@ -426,27 +426,27 @@ Stage-0 predicate:value is not null TableScan [TS_28] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] + <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_24] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_158] (rows=67/61 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 16 [SIMPLE_EDGE] + <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_17] - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_157] (rows=42/52 width=86) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 Select Operator [SEL_11] (rows=525/319 width=178) @@ -454,7 +454,7 @@ Stage-0 Group By Operator [GBY_10] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_2] (rows=25/25 width=175) @@ -463,7 +463,7 @@ Stage-0 predicate:value is not null TableScan [TS_0] (rows=25/25 width=175) Output:["key","value"] - <-Map 10 [CONTAINS] + <-Map 10 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_5] (rows=500/500 width=178) @@ -627,7 +627,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 + Map 1 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=253/480 width=190) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -669,13 +669,13 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 llap File Output Operator [FS_16] Select Operator [SEL_15] (rows=278/1166 width=3) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_27] (rows=278/1166 width=3) Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_25] (rows=253/480 width=3) @@ -692,7 +692,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] + <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=230/242 width=175) @@ -728,7 +728,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Map 1 + Map 1 llap File Output Operator [FS_10] Merge Join Operator [MERGEJOIN_15] (rows=253/480 width=190) Conds:SEL_2._col0=SEL_5._col0(Inner),Output:["_col0","_col1"] @@ -774,13 +774,13 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 llap File Output Operator [FS_16] Select Operator [SEL_15] (rows=278/1166 width=3) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_27] (rows=278/1166 width=3) Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1","_col2"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_25] (rows=253/480 width=3) @@ -797,7 +797,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) default@tab2,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] + <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=230/242 width=175) @@ -851,15 +851,15 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 + Reducer 4 llap File Output Operator [FS_25] Group By Operator [GBY_23] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Reducer 3 [CUSTOM_SIMPLE_EDGE] + <-Reducer 3 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_22] Merge Join Operator [MERGEJOIN_38] (rows=531/1646 width=3) Conds:Union 2._col0=RS_19._col0(Inner) - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col0 Select Operator [SEL_17] (rows=475/500 width=3) @@ -869,7 +869,7 @@ Stage-0 TableScan [TS_15] (rows=500/500 width=3) default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_18] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_36] (rows=253/480 width=3) @@ -886,7 +886,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) Output:["key"] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_18] PartitionCols:_col0 Select Operator [SEL_12] (rows=230/242 width=3) @@ -940,15 +940,15 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 5 + Reducer 5 llap File Output Operator [FS_31] Group By Operator [GBY_29] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Reducer 4 [CUSTOM_SIMPLE_EDGE] + <-Reducer 4 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_28] Merge Join Operator [MERGEJOIN_50] (rows=558/3768 width=3) Conds:Union 3._col0=RS_25._col0(Inner) - <-Map 9 [SIMPLE_EDGE] + <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_25] PartitionCols:_col0 Select Operator [SEL_23] (rows=475/500 width=3) @@ -958,7 +958,7 @@ Stage-0 TableScan [TS_21] (rows=500/500 width=3) default@tab_part,b,Tbl:COMPLETE,Col:NONE,Output:["key"] <-Union 3 [SIMPLE_EDGE] - <-Map 8 [CONTAINS] + <-Map 8 [CONTAINS] llap Reduce Output Operator [RS_24] PartitionCols:_col0 Select Operator [SEL_18] (rows=230/242 width=3) @@ -967,14 +967,14 @@ Stage-0 predicate:key is not null TableScan [TS_16] (rows=242/242 width=3) Output:["key"] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_24] PartitionCols:_col0 Select Operator [SEL_15] (rows=278/1166 width=3) Output:["_col0"] Merge Join Operator [MERGEJOIN_49] (rows=278/1166 width=3) Conds:RS_12._col2=RS_13._col1(Inner),Output:["_col1"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_12] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_47] (rows=253/480 width=3) @@ -991,7 +991,7 @@ Stage-0 predicate:key is not null TableScan [TS_0] (rows=242/242 width=3) default@tab,s3,Tbl:COMPLETE,Col:NONE,Output:["key"] - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_13] PartitionCols:_col1 Select Operator [SEL_8] (rows=230/242 width=175) @@ -1137,14 +1137,14 @@ Stage-5 Dependency Collection{} Stage-3 Union 5 - <-Reducer 12 [CONTAINS] + <-Reducer 12 [CONTAINS] llap File Output Operator [FS_75] table:{"name:":"default.a"} Select Operator [SEL_44] (rows=2682/5421 width=178) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_122] (rows=2682/5421 width=178) Conds:RS_41._col1=RS_42._col0(Inner),Output:["_col1","_col4"] - <-Map 17 [SIMPLE_EDGE] + <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_42] PartitionCols:_col0 Select Operator [SEL_37] (rows=500/500 width=178) @@ -1153,12 +1153,12 @@ Stage-5 predicate:key is not null TableScan [TS_35] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 11 [SIMPLE_EDGE] + <-Reducer 11 [SIMPLE_EDGE] llap SHUFFLE [RS_41] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_121] (rows=1658/2097 width=87) Conds:Union 14._col0=RS_39._col1(Inner),Output:["_col1"] - <-Map 10 [SIMPLE_EDGE] + <-Map 10 [SIMPLE_EDGE] llap SHUFFLE [RS_39] PartitionCols:_col1 Select Operator [SEL_34] (rows=500/500 width=178) @@ -1168,7 +1168,7 @@ Stage-5 TableScan [TS_11] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Union 14 [SIMPLE_EDGE] - <-Map 13 [CONTAINS] + <-Map 13 [CONTAINS] llap Reduce Output Operator [RS_38] PartitionCols:_col0 Select Operator [SEL_23] (rows=25/25 width=89) @@ -1177,7 +1177,7 @@ Stage-5 predicate:value is not null TableScan [TS_21] (rows=25/25 width=89) Output:["value"] - <-Map 15 [CONTAINS] + <-Map 15 [CONTAINS] llap Reduce Output Operator [RS_38] PartitionCols:_col0 Select Operator [SEL_26] (rows=500/500 width=91) @@ -1186,7 +1186,7 @@ Stage-5 predicate:value is not null TableScan [TS_24] (rows=500/500 width=91) Output:["value"] - <-Map 16 [CONTAINS] + <-Map 16 [CONTAINS] llap Reduce Output Operator [RS_38] PartitionCols:_col0 Select Operator [SEL_30] (rows=500/500 width=91) @@ -1201,14 +1201,14 @@ Stage-5 File Output Operator [FS_79] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_44] - <-Reducer 4 [CONTAINS] + <-Reducer 4 [CONTAINS] llap File Output Operator [FS_75] table:{"name:":"default.a"} Select Operator [SEL_20] (rows=67/170 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_120] (rows=67/170 width=177) Conds:RS_17._col1=RS_18._col0(Inner),Output:["_col1","_col4"] - <-Map 10 [SIMPLE_EDGE] + <-Map 10 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col0 Select Operator [SEL_13] (rows=500/500 width=178) @@ -1216,12 +1216,12 @@ Stage-5 Filter Operator [FIL_107] (rows=500/500 width=178) predicate:key is not null Please refer to the previous TableScan [TS_11] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_17] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_119] (rows=42/108 width=86) Conds:Union 2._col0=RS_15._col1(Inner),Output:["_col1"] - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_15] PartitionCols:_col1 Select Operator [SEL_10] (rows=25/25 width=175) @@ -1231,7 +1231,7 @@ Stage-5 TableScan [TS_8] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_14] PartitionCols:_col0 Select Operator [SEL_2] (rows=25/25 width=89) @@ -1240,7 +1240,7 @@ Stage-5 predicate:value is not null TableScan [TS_0] (rows=25/25 width=89) Output:["value"] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_14] PartitionCols:_col0 Select Operator [SEL_5] (rows=500/500 width=91) @@ -1255,23 +1255,23 @@ Stage-5 File Output Operator [FS_79] table:{"name:":"default.c"} Please refer to the previous Select Operator [SEL_20] - <-Reducer 9 [CONTAINS] + <-Reducer 9 [CONTAINS] llap File Output Operator [FS_75] table:{"name:":"default.a"} Select Operator [SEL_72] (rows=192/820 width=175) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_124] (rows=192/820 width=175) Conds:RS_69._col1=Union 20._col0(Inner),Output:["_col0","_col3"] - <-Reducer 8 [SIMPLE_EDGE] + <-Reducer 8 [SIMPLE_EDGE] llap SHUFFLE [RS_69] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_123] (rows=39/115 width=264) Conds:RS_66._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 7 [SIMPLE_EDGE] + <-Map 7 [SIMPLE_EDGE] llap SHUFFLE [RS_66] PartitionCols:_col0 Please refer to the previous Select Operator [SEL_10] - <-Map 18 [SIMPLE_EDGE] + <-Map 18 [SIMPLE_EDGE] llap SHUFFLE [RS_67] PartitionCols:_col0 Select Operator [SEL_51] (rows=25/25 width=175) @@ -1281,7 +1281,7 @@ Stage-5 TableScan [TS_49] (rows=25/25 width=175) default@src1,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Union 20 [SIMPLE_EDGE] - <-Map 19 [CONTAINS] + <-Map 19 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_54] (rows=25/25 width=89) @@ -1290,7 +1290,7 @@ Stage-5 predicate:value is not null TableScan [TS_52] (rows=25/25 width=89) Output:["value"] - <-Map 21 [CONTAINS] + <-Map 21 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_57] (rows=500/500 width=91) @@ -1299,7 +1299,7 @@ Stage-5 predicate:value is not null TableScan [TS_55] (rows=500/500 width=91) Output:["value"] - <-Map 22 [CONTAINS] + <-Map 22 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_61] (rows=500/500 width=91) @@ -1308,7 +1308,7 @@ Stage-5 predicate:value is not null TableScan [TS_59] (rows=500/500 width=91) Output:["value"] - <-Map 23 [CONTAINS] + <-Map 23 [CONTAINS] llap Reduce Output Operator [RS_70] PartitionCols:_col0 Select Operator [SEL_64] (rows=500/500 width=91) @@ -1443,25 +1443,25 @@ Stage-5 Stage-4 Dependency Collection{} Stage-3 - Reducer 9 + Reducer 9 llap File Output Operator [FS_115] table:{"name:":"default.a"} Group By Operator [GBY_112] (rows=2941/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 8 [SIMPLE_EDGE] - <-Reducer 13 [CONTAINS] + <-Reducer 13 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Select Operator [SEL_107] (rows=192/304 width=175) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_164] (rows=192/304 width=175) Conds:RS_104._col1=RS_105._col1(Inner),Output:["_col0","_col3"] - <-Reducer 12 [SIMPLE_EDGE] + <-Reducer 12 [SIMPLE_EDGE] llap SHUFFLE [RS_104] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_163] (rows=39/115 width=264) Conds:RS_101._col0=RS_102._col0(Inner),Output:["_col0","_col1","_col3"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_101] PartitionCols:_col0 Select Operator [SEL_14] (rows=25/25 width=175) @@ -1470,7 +1470,7 @@ Stage-5 predicate:(key is not null and value is not null) TableScan [TS_12] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 25 [SIMPLE_EDGE] + <-Map 25 [SIMPLE_EDGE] llap SHUFFLE [RS_102] PartitionCols:_col0 Select Operator [SEL_70] (rows=25/25 width=175) @@ -1479,7 +1479,7 @@ Stage-5 predicate:key is not null TableScan [TS_68] (rows=25/25 width=175) default@src1,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 32 [SIMPLE_EDGE] + <-Reducer 32 [SIMPLE_EDGE] llap SHUFFLE [RS_105] PartitionCols:_col1 Select Operator [SEL_100] (rows=1525/319 width=178) @@ -1487,7 +1487,7 @@ Stage-5 Group By Operator [GBY_99] (rows=1525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 31 [SIMPLE_EDGE] - <-Map 35 [CONTAINS] + <-Map 35 [CONTAINS] llap Reduce Output Operator [RS_98] PartitionCols:_col1, _col0 Select Operator [SEL_94] (rows=500/500 width=178) @@ -1496,7 +1496,7 @@ Stage-5 predicate:value is not null TableScan [TS_92] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 30 [CONTAINS] + <-Reducer 30 [CONTAINS] llap Reduce Output Operator [RS_98] PartitionCols:_col1, _col0 Select Operator [SEL_91] (rows=1025/319 width=178) @@ -1504,7 +1504,7 @@ Stage-5 Group By Operator [GBY_90] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 29 [SIMPLE_EDGE] - <-Map 34 [CONTAINS] + <-Map 34 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col1, _col0 Select Operator [SEL_85] (rows=500/500 width=178) @@ -1513,7 +1513,7 @@ Stage-5 predicate:value is not null TableScan [TS_83] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 28 [CONTAINS] + <-Reducer 28 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col1, _col0 Select Operator [SEL_82] (rows=525/319 width=178) @@ -1521,7 +1521,7 @@ Stage-5 Group By Operator [GBY_81] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 27 [SIMPLE_EDGE] - <-Map 26 [CONTAINS] + <-Map 26 [CONTAINS] llap Reduce Output Operator [RS_80] PartitionCols:_col1, _col0 Select Operator [SEL_73] (rows=25/25 width=175) @@ -1530,7 +1530,7 @@ Stage-5 predicate:value is not null TableScan [TS_71] (rows=25/25 width=175) Output:["key","value"] - <-Map 33 [CONTAINS] + <-Map 33 [CONTAINS] llap Reduce Output Operator [RS_80] PartitionCols:_col1, _col0 Select Operator [SEL_76] (rows=500/500 width=178) @@ -1539,20 +1539,20 @@ Stage-5 predicate:value is not null TableScan [TS_74] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 7 [CONTAINS] + <-Reducer 7 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 Group By Operator [GBY_63] (rows=2749/309 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] - <-Reducer 16 [CONTAINS] + <-Reducer 16 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_58] (rows=2682/1056 width=178) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_162] (rows=2682/1056 width=178) Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] - <-Map 24 [SIMPLE_EDGE] + <-Map 24 [SIMPLE_EDGE] llap SHUFFLE [RS_56] PartitionCols:_col0 Select Operator [SEL_51] (rows=500/500 width=178) @@ -1561,12 +1561,12 @@ Stage-5 predicate:key is not null TableScan [TS_49] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 15 [SIMPLE_EDGE] + <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_161] (rows=1658/512 width=87) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] - <-Map 14 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] llap SHUFFLE [RS_53] PartitionCols:_col1 Select Operator [SEL_48] (rows=500/500 width=178) @@ -1575,7 +1575,7 @@ Stage-5 predicate:(key is not null and value is not null) TableScan [TS_15] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Reducer 21 [SIMPLE_EDGE] + <-Reducer 21 [SIMPLE_EDGE] llap SHUFFLE [RS_52] PartitionCols:_col1 Select Operator [SEL_45] (rows=1025/319 width=178) @@ -1583,7 +1583,7 @@ Stage-5 Group By Operator [GBY_44] (rows=1025/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 20 [SIMPLE_EDGE] - <-Map 23 [CONTAINS] + <-Map 23 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_39] (rows=500/500 width=178) @@ -1592,7 +1592,7 @@ Stage-5 predicate:value is not null TableScan [TS_37] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 19 [CONTAINS] + <-Reducer 19 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 Select Operator [SEL_36] (rows=525/319 width=178) @@ -1600,7 +1600,7 @@ Stage-5 Group By Operator [GBY_35] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 18 [SIMPLE_EDGE] - <-Map 17 [CONTAINS] + <-Map 17 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_27] (rows=25/25 width=175) @@ -1609,7 +1609,7 @@ Stage-5 predicate:value is not null TableScan [TS_25] (rows=25/25 width=175) Output:["key","value"] - <-Map 22 [CONTAINS] + <-Map 22 [CONTAINS] llap Reduce Output Operator [RS_34] PartitionCols:_col1, _col0 Select Operator [SEL_30] (rows=500/500 width=178) @@ -1618,14 +1618,14 @@ Stage-5 predicate:value is not null TableScan [TS_28] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 5 [CONTAINS] + <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 Select Operator [SEL_24] (rows=67/61 width=177) Output:["_col0","_col1"] Merge Join Operator [MERGEJOIN_160] (rows=67/61 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] - <-Map 14 [SIMPLE_EDGE] + <-Map 14 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col0 Select Operator [SEL_17] (rows=500/500 width=178) @@ -1633,16 +1633,16 @@ Stage-5 Filter Operator [FIL_147] (rows=500/500 width=178) predicate:key is not null Please refer to the previous TableScan [TS_15] - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_159] (rows=42/52 width=86) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] - <-Map 11 [SIMPLE_EDGE] + <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_19] PartitionCols:_col1 Please refer to the previous Select Operator [SEL_14] - <-Reducer 3 [SIMPLE_EDGE] + <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 Select Operator [SEL_11] (rows=525/319 width=178) @@ -1650,7 +1650,7 @@ Stage-5 Group By Operator [GBY_10] (rows=525/319 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] - <-Map 1 [CONTAINS] + <-Map 1 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_2] (rows=25/25 width=175) @@ -1659,7 +1659,7 @@ Stage-5 predicate:value is not null TableScan [TS_0] (rows=25/25 width=175) Output:["key","value"] - <-Map 10 [CONTAINS] + <-Map 10 [CONTAINS] llap Reduce Output Operator [RS_9] PartitionCols:_col1, _col0 Select Operator [SEL_5] (rows=500/500 width=178) @@ -1751,34 +1751,34 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 5 + Reducer 5 llap File Output Operator [FS_18] table:{"name:":"default.dest1"} Select Operator [SEL_16] (rows=309/310 width=272) Output:["_col0","_col1"] Group By Operator [GBY_15] (rows=309/310 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 - <-Reducer 4 [SIMPLE_EDGE] + <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_14] PartitionCols:_col0 Group By Operator [GBY_11] (rows=501/310 width=272) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Select Operator [SEL_6] (rows=500/500 width=178) Output:["_col0","_col1"] TableScan [TS_5] (rows=500/500 width=178) Output:["key","value"] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0, _col1 Select Operator [SEL_4] (rows=1/1 width=272) Output:["_col0","_col1"] Group By Operator [GBY_3] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] Select Operator [SEL_1] (rows=500/500 width=10) TableScan [TS_0] (rows=500/500 width=10) @@ -1830,86 +1830,28 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 llap File Output Operator [FS_8] Select Operator [SEL_7] (rows=261/4122 width=260) Output:["_col0","_col1","_col2"] Merge Join Operator [MERGEJOIN_9] (rows=261/4122 width=260) Conds:RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),RS_3.key=RS_3.key(Unique),Output:["_col0","_col5","_col10"] - <-Map 1 [SIMPLE_EDGE] + <-Map 1 [SIMPLE_EDGE] llap SHUFFLE [RS_3] PartitionCols:key TableScan [TS_0] (rows=500/500 width=87) default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [SIMPLE_EDGE] + <-Map 3 [SIMPLE_EDGE] llap SHUFFLE [RS_4] PartitionCols:key TableScan [TS_1] (rows=25/25 width=86) default@src1,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 4 [SIMPLE_EDGE] + <-Map 4 [SIMPLE_EDGE] llap SHUFFLE [RS_5] PartitionCols:key TableScan [TS_2] (rows=2000/2000 width=87) default@srcpart,c,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] -PREHOOK: query: SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -PREHOOK: type: QUERY -PREHOOK: Input: cat -PREHOOK: Input: default@src -#### A masked pattern was here #### -POSTHOOK: query: SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -POSTHOOK: type: QUERY -POSTHOOK: Input: cat -POSTHOOK: Input: default@src -#### A masked pattern was here #### -PREHOOK: query: explain analyze -SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -PREHOOK: type: QUERY -POSTHOOK: query: explain analyze -SELECT -TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) -FROM src a join src b -on a.key = b.key -POSTHOOK: type: QUERY -Plan not optimized by CBO. - -Vertex dependency in root stage -Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) - -Stage-0 - Fetch Operator - limit:-1 - Stage-1 - Reducer 2 - File Output Operator [FS_9] - Transform Operator [SCR_8] (rows=809/1028 width=178) - command:cat - Merge Join Operator [MERGEJOIN_14] (rows=809/1028 width=178) - Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] - <-Map 1 [SIMPLE_EDGE] - SHUFFLE [RS_3] - PartitionCols:key - Filter Operator [FIL_12] (rows=500/500 width=178) - predicate:key is not null - TableScan [TS_0] (rows=500/500 width=178) - default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Map 3 [SIMPLE_EDGE] - SHUFFLE [RS_5] - PartitionCols:key - Filter Operator [FIL_13] (rows=500/500 width=87) - predicate:key is not null - TableScan [TS_1] (rows=500/500 width=87) - default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - PREHOOK: query: FROM ( select key, value from ( select 'tst1' as key, cast(count(1) as string) as value, 'tst1' as value2 from src s1 @@ -1983,7 +1925,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 + Reducer 4 llap File Output Operator [FS_16] table:{"name:":"default.dest1"} Select Operator [SEL_14] (rows=309/310 width=272) @@ -1991,7 +1933,7 @@ Stage-4 Group By Operator [GBY_13] (rows=309/310 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_12] PartitionCols:_col0 Select Operator [SEL_6] (rows=500/500 width=178) @@ -2001,7 +1943,7 @@ Stage-4 Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_6] - <-Map 7 [CONTAINS] + <-Map 7 [CONTAINS] llap Reduce Output Operator [RS_12] PartitionCols:_col0 Select Operator [SEL_9] (rows=500/500 width=178) @@ -2011,14 +1953,14 @@ Stage-4 Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_9] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_12] PartitionCols:_col0 Select Operator [SEL_4] (rows=1/1 width=272) Output:["_col0","_col1"] Group By Operator [GBY_3] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] Select Operator [SEL_1] (rows=500/500 width=10) TableScan [TS_0] (rows=500/500 width=10) @@ -2026,7 +1968,7 @@ Stage-4 Reduce Output Operator [RS_18] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_4] - Reducer 5 + Reducer 5 llap File Output Operator [FS_22] table:{"name:":"default.dest2"} Select Operator [SEL_20] (rows=1001/310 width=456) @@ -2101,7 +2043,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 4 + Reducer 4 llap File Output Operator [FS_14] table:{"name:":"default.dest1"} Select Operator [SEL_12] (rows=309/310 width=272) @@ -2109,7 +2051,7 @@ Stage-4 Group By Operator [GBY_11] (rows=309/310 width=96) Output:["_col0","_col1"],aggregations:["count(DISTINCT KEY._col1:0._col0)"],keys:KEY._col0 <-Union 3 [SIMPLE_EDGE] - <-Map 6 [CONTAINS] + <-Map 6 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0 Select Operator [SEL_6] (rows=500/500 width=178) @@ -2119,14 +2061,14 @@ Stage-4 Reduce Output Operator [RS_16] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_6] - <-Reducer 2 [CONTAINS] + <-Reducer 2 [CONTAINS] llap Reduce Output Operator [RS_10] PartitionCols:_col0 Select Operator [SEL_4] (rows=1/1 width=272) Output:["_col0","_col1"] Group By Operator [GBY_3] (rows=1/1 width=8) Output:["_col0"],aggregations:["count()"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] + <-Map 1 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] Select Operator [SEL_1] (rows=500/500 width=10) TableScan [TS_0] (rows=500/500 width=10) @@ -2134,7 +2076,7 @@ Stage-4 Reduce Output Operator [RS_16] PartitionCols:_col0, _col1 Please refer to the previous Select Operator [SEL_4] - Reducer 5 + Reducer 5 llap File Output Operator [FS_20] table:{"name:":"default.dest2"} Select Operator [SEL_18] (rows=501/310 width=456) diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out index 3b3983f896..dc4e55478a 100644 --- a/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_2.q.out @@ -34,82 +34,6 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@u3 POSTHOOK: Lineage: u3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: u3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from u1 -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -#### A masked pattern was here #### -POSTHOOK: query: select * from u1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: select * from u2 -PREHOOK: type: QUERY -PREHOOK: Input: default@u2 -#### A masked pattern was here #### -POSTHOOK: query: select * from u2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u2 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -PREHOOK: query: select * from u3 -PREHOOK: type: QUERY -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select * from u3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 PREHOOK: query: select key, value from ( select key, value from u1 @@ -244,38 +168,6 @@ POSTHOOK: Input: default@u3 98 val_98 PREHOOK: query: select distinct * from ( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select distinct * from -( select distinct * from u1 union select key, value from u2 diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out new file mode 100644 index 0000000000..96c212477c --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out @@ -0,0 +1,2650 @@ +PREHOOK: query: explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain + select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select count(1) FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: drop table if exists tmptable +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists tmptable +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tmptable(key string, value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tmptable +POSTHOOK: query: create table tmptable(key string, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table tmptable + select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 5 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + Union 3 + Vertex: Union 3 + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.tmptable + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@tmptable +POSTHOOK: query: insert overwrite table tmptable +select unionsrc.key, unionsrc.value FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@tmptable +POSTHOOK: Lineage: tmptable.key EXPRESSION [(src1)s2.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tmptable.value EXPRESSION [(src)s1.null, (src1)s2.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select * from tmptable x sort by x.key, x.value +PREHOOK: type: QUERY +PREHOOK: Input: default@tmptable +#### A masked pattern was here #### +POSTHOOK: query: select * from tmptable x sort by x.key, x.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tmptable +#### A masked pattern was here #### + + val_165 + val_193 + val_265 + val_27 + val_409 + val_484 +128 +146 val_146 +150 val_150 +213 val_213 +224 +238 val_238 +255 val_255 +273 val_273 +278 val_278 +311 val_311 +369 +401 val_401 +406 val_406 +66 val_66 +98 val_98 +tst1 500 +PREHOOK: query: drop table if exists tmptable +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tmptable +PREHOOK: Output: default@tmptable +POSTHOOK: query: drop table if exists tmptable +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tmptable +POSTHOOK: Output: default@tmptable +PREHOOK: query: explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 6 <- Union 2 (CONTAINS) + Map 7 <- Union 4 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, unionsrc.value FROM (select s1.key as key, s1.value as value from src s1 UNION DISTINCT + select s2.key as key, s2.value as value from src s2 UNION DISTINCT + select s3.key as key, s3.value as value from src s3) unionsrc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 +103 val_103 +105 val_105 +116 val_116 +120 val_120 +129 val_129 +131 val_131 +136 val_136 +155 val_155 +157 val_157 +160 val_160 +180 val_180 +183 val_183 +190 val_190 +191 val_191 +193 val_193 +205 val_205 +207 val_207 +213 val_213 +223 val_223 +235 val_235 +238 val_238 +239 val_239 +24 val_24 +248 val_248 +260 val_260 +28 val_28 +286 val_286 +298 val_298 +305 val_305 +306 val_306 +307 val_307 +308 val_308 +315 val_315 +317 val_317 +321 val_321 +33 val_33 +331 val_331 +356 val_356 +360 val_360 +366 val_366 +367 val_367 +368 val_368 +37 val_37 +378 val_378 +379 val_379 +382 val_382 +389 val_389 +395 val_395 +400 val_400 +402 val_402 +406 val_406 +407 val_407 +42 val_42 +430 val_430 +436 val_436 +44 val_44 +452 val_452 +466 val_466 +467 val_467 +472 val_472 +480 val_480 +484 val_484 +485 val_485 +487 val_487 +495 val_495 +496 val_496 +498 val_498 +51 val_51 +57 val_57 +58 val_58 +65 val_65 +66 val_66 +69 val_69 +70 val_70 +78 val_78 +8 val_8 +92 val_92 +96 val_96 +104 val_104 +111 val_111 +114 val_114 +118 val_118 +12 val_12 +133 val_133 +138 val_138 +153 val_153 +156 val_156 +162 val_162 +168 val_168 +174 val_174 +176 val_176 +178 val_178 +181 val_181 +186 val_186 +187 val_187 +192 val_192 +197 val_197 +199 val_199 +2 val_2 +201 val_201 +203 val_203 +208 val_208 +214 val_214 +218 val_218 +219 val_219 +230 val_230 +237 val_237 +249 val_249 +256 val_256 +257 val_257 +277 val_277 +278 val_278 +284 val_284 +285 val_285 +310 val_310 +322 val_322 +327 val_327 +333 val_333 +336 val_336 +338 val_338 +341 val_341 +344 val_344 +348 val_348 +351 val_351 +362 val_362 +364 val_364 +374 val_374 +392 val_392 +397 val_397 +4 val_4 +403 val_403 +404 val_404 +419 val_419 +421 val_421 +427 val_427 +429 val_429 +437 val_437 +438 val_438 +439 val_439 +454 val_454 +455 val_455 +460 val_460 +463 val_463 +470 val_470 +478 val_478 +479 val_479 +489 val_489 +491 val_491 +492 val_492 +53 val_53 +67 val_67 +82 val_82 +90 val_90 +97 val_97 +10 val_10 +125 val_125 +134 val_134 +143 val_143 +145 val_145 +149 val_149 +150 val_150 +158 val_158 +163 val_163 +164 val_164 +166 val_166 +17 val_17 +170 val_170 +172 val_172 +189 val_189 +19 val_19 +195 val_195 +196 val_196 +20 val_20 +209 val_209 +216 val_216 +217 val_217 +224 val_224 +228 val_228 +229 val_229 +233 val_233 +241 val_241 +244 val_244 +247 val_247 +255 val_255 +258 val_258 +26 val_26 +263 val_263 +265 val_265 +266 val_266 +272 val_272 +273 val_273 +274 val_274 +281 val_281 +291 val_291 +296 val_296 +30 val_30 +302 val_302 +309 val_309 +316 val_316 +318 val_318 +325 val_325 +332 val_332 +335 val_335 +339 val_339 +342 val_342 +345 val_345 +353 val_353 +369 val_369 +373 val_373 +375 val_375 +377 val_377 +384 val_384 +386 val_386 +394 val_394 +396 val_396 +399 val_399 +401 val_401 +41 val_41 +413 val_413 +414 val_414 +431 val_431 +446 val_446 +448 val_448 +449 val_449 +459 val_459 +462 val_462 +468 val_468 +47 val_47 +482 val_482 +490 val_490 +493 val_493 +494 val_494 +497 val_497 +5 val_5 +54 val_54 +74 val_74 +77 val_77 +80 val_80 +84 val_84 +85 val_85 +87 val_87 +9 val_9 +95 val_95 +100 val_100 +11 val_11 +113 val_113 +119 val_119 +126 val_126 +128 val_128 +137 val_137 +146 val_146 +15 val_15 +152 val_152 +165 val_165 +167 val_167 +169 val_169 +175 val_175 +177 val_177 +179 val_179 +18 val_18 +194 val_194 +200 val_200 +202 val_202 +221 val_221 +222 val_222 +226 val_226 +242 val_242 +252 val_252 +262 val_262 +27 val_27 +275 val_275 +280 val_280 +282 val_282 +283 val_283 +287 val_287 +288 val_288 +289 val_289 +292 val_292 +311 val_311 +323 val_323 +34 val_34 +35 val_35 +365 val_365 +393 val_393 +409 val_409 +411 val_411 +417 val_417 +418 val_418 +424 val_424 +43 val_43 +432 val_432 +435 val_435 +443 val_443 +444 val_444 +453 val_453 +457 val_457 +458 val_458 +469 val_469 +475 val_475 +477 val_477 +481 val_481 +483 val_483 +64 val_64 +72 val_72 +76 val_76 +83 val_83 +86 val_86 +98 val_98 +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) + Reducer 7 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 9 <- Map 8 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: bigint) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string), _col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: bigint) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: bigint) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION DISTINCT + select 'tst2' as key, count(1) as value from src s2 + UNION DISTINCT + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +tst1 1 +tst2 1 +tst3 1 +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (CUSTOM_SIMPLE_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 4 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 13 Data size: 1248 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select 'tst1' as key, cast(count(1) as string) as value from src s1) + unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 7 <- Union 3 (CONTAINS) + Map 8 <- Union 5 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE), Union 5 (CONTAINS) + Reducer 6 <- Union 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 7 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: s3 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 13 Data size: 3536 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 19 Data size: 5168 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 16 Data size: 1536 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + Union 5 + Vertex: Union 5 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +#### A masked pattern was here #### +POSTHOOK: query: select unionsrc.key, count(1) FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src1 s2 + UNION DISTINCT + select s3.key as key, s3.value as value from src1 s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +#### A masked pattern was here #### + 7 +128 1 +146 1 +150 1 +213 1 +224 1 +238 1 +255 1 +273 1 +278 1 +311 1 +369 1 +401 1 +406 1 +66 1 +98 1 +tst1 1 +PREHOOK: query: EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 11 <- Union 2 (CONTAINS) + Map 12 <- Union 4 (CONTAINS) + Map 13 <- Union 6 (CONTAINS) + Map 14 <- Union 8 (CONTAINS) + Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 12 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 13 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 14 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(1) FROM ( + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src UNION DISTINCT + SELECT key, value FROM src) src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +309 +PREHOOK: query: explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 10 <- Union 7 (CONTAINS) + Map 9 <- Union 3 (CONTAINS) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE) + Reducer 6 <- Map 1 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS) + Reducer 8 <- Union 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map 10 + Map Operator Tree: + TableScan + alias: s4 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 9 + Map Operator Tree: + TableScan + alias: s2 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (UDFToDouble(key) < 10.0) (type: boolean) + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 66 Data size: 35904 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 66 Data size: 35904 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string), UDFToString(_col0) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 272 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string), _col1 (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 8 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string), KEY._col1 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 83 Data size: 22576 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Union 3 + Vertex: Union 3 + Union 7 + Vertex: Union 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT unionsrc1.key, unionsrc1.value, unionsrc2.key, unionsrc2.value +FROM (select 'tst1' as key, cast(count(1) as string) as value from src s1 + UNION DISTINCT + select s2.key as key, s2.value as value from src s2 where s2.key < 10) unionsrc1 +JOIN + (select 'tst1' as key, cast(count(1) as string) as value from src s3 + UNION DISTINCT + select s4.key as key, s4.value as value from src s4 where s4.key < 10) unionsrc2 +ON (unionsrc1.key = unionsrc2.key) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 val_0 0 val_0 +2 val_2 2 val_2 +4 val_4 4 val_4 +5 val_5 5 val_5 +8 val_8 8 val_8 +9 val_9 9 val_9 +tst1 500 tst1 500 +PREHOOK: query: explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 10 <- Union 2 (CONTAINS) + Map 11 <- Union 4 (CONTAINS) + Map 12 <- Union 6 (CONTAINS) + Map 13 <- Union 8 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 4 (CONTAINS) + Reducer 5 <- Union 4 (SIMPLE_EDGE), Union 6 (CONTAINS) + Reducer 7 <- Union 6 (SIMPLE_EDGE), Union 8 (CONTAINS) + Reducer 9 <- Union 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: '1' (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 42500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 10 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: reverse(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 11 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: llap + LLAP IO: no inputs + Map 12 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: astring (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: no inputs + Map 13 + Map Operator Tree: + TableScan + alias: src_thrift + Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: lstring[0] (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 11 Data size: 21120 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 309 Data size: 56856 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 7 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 160 Data size: 29440 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 85 Data size: 15640 Basic stats: COMPLETE Column stats: PARTIAL + Group By Operator + aggregations: count(1) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 85 Data size: 16320 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 85 Data size: 16320 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 4 + Vertex: Union 4 + Union 6 + Vertex: Union 6 + Union 8 + Vertex: Union 8 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src_thrift +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, count(1) +FROM ( + SELECT '1' as key from src + UNION DISTINCT + SELECT reverse(key) as key from src + UNION DISTINCT + SELECT key as key from src + UNION DISTINCT + SELECT astring as key from src_thrift + UNION DISTINCT + SELECT lstring[0] as key from src_thrift +) union_output +GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src_thrift +#### A masked pattern was here #### +03 1 +071 1 +09 1 +1 1 +10 1 +100 1 +104 1 +111 1 +114 1 +116 1 +123 1 +126 1 +128 1 +131 1 +134 1 +14 1 +152 1 +155 1 +169 1 +17 1 +170 1 +172 1 +178 1 +179 1 +182 1 +189 1 +19 1 +192 1 +194 1 +195 1 +199 1 +20 1 +207 1 +21 1 +213 1 +218 1 +223 1 +229 1 +233 1 +235 1 +24 1 +256 1 +257 1 +26 1 +265 1 +266 1 +27 1 +278 1 +280 1 +284 1 +291 1 +294 1 +30 1 +301 1 +302 1 +305 1 +312 1 +314 1 +316 1 +321 1 +323 1 +33 1 +333 1 +338 1 +344 1 +353 1 +356 1 +361 1 +364 1 +368 1 +369 1 +37 1 +377 1 +378 1 +386 1 +392 1 +397 1 +40 1 +404 1 +407 1 +411 1 +418 1 +419 1 +422 1 +424 1 +430 1 +432 1 +436 1 +437 1 +448 1 +454 1 +457 1 +46 1 +468 1 +477 1 +479 1 +490 1 +492 1 +493 1 +495 1 +496 1 +497 1 +501 1 +533 1 +534 1 +552 1 +57 1 +582 1 +611 1 +62 1 +652 1 +67 1 +671 1 +691 1 +712 1 +723 1 +73 1 +732 1 +75 1 +751 1 +761 1 +794 1 +8 1 +80 1 +814 1 +822 1 +831 1 +85 1 +854 1 +863 1 +871 1 +874 1 +882 1 +9 1 +902 1 +904 1 +91 1 +911 1 +92 1 +921 1 +922 1 +964 1 +97 1 +974 1 +982 1 +983 1 +record_7 1 +record_8 1 +NULL 1 +002 1 +02 1 +032 1 +034 1 +07 1 +074 1 +091 1 +105 1 +11 1 +113 1 +124 1 +136 1 +137 1 +138 1 +143 1 +150 1 +158 1 +160 1 +162 1 +165 1 +166 1 +174 1 +177 1 +18 1 +180 1 +186 1 +196 1 +197 1 +204 1 +217 1 +219 1 +226 1 +230 1 +238 1 +242 1 +248 1 +252 1 +260 1 +261 1 +263 1 +274 1 +277 1 +293 1 +296 1 +304 1 +307 1 +308 1 +325 1 +327 1 +342 1 +348 1 +35 1 +351 1 +367 1 +381 1 +389 1 +393 1 +394 1 +4 1 +403 1 +409 1 +412 1 +414 1 +421 1 +427 1 +429 1 +435 1 +438 1 +439 1 +45 1 +455 1 +458 1 +460 1 +461 1 +462 1 +471 1 +475 1 +478 1 +480 1 +484 1 +487 1 +50 1 +503 1 +513 1 +521 1 +523 1 +53 1 +532 1 +551 1 +56 1 +563 1 +571 1 +573 1 +58 1 +59 1 +591 1 +603 1 +604 1 +622 1 +633 1 +634 1 +641 1 +644 1 +651 1 +66 1 +661 1 +69 1 +693 1 +694 1 +703 1 +71 1 +731 1 +734 1 +752 1 +754 1 +764 1 +77 1 +772 1 +78 1 +784 1 +793 1 +803 1 +813 1 +83 1 +833 1 +843 1 +844 1 +864 1 +89 1 +90 1 +924 1 +934 1 +95 1 +954 1 +961 1 +993 1 +record_0 1 +record_2 1 +record_4 1 +record_6 1 +record_9 1 +001 1 +004 1 +01 1 +013 1 +051 1 +061 1 +062 1 +063 1 +102 1 +103 1 +120 1 +129 1 +133 1 +142 1 +145 1 +146 1 +149 1 +15 1 +157 1 +181 1 +187 1 +191 1 +2 1 +200 1 +201 1 +202 1 +203 1 +208 1 +214 1 +216 1 +222 1 +224 1 +228 1 +237 1 +239 1 +241 1 +244 1 +254 1 +255 1 +258 1 +262 1 +264 1 +273 1 +275 1 +28 1 +282 1 +285 1 +286 1 +287 1 +288 1 +29 1 +292 1 +298 1 +310 1 +311 1 +332 1 +335 1 +336 1 +339 1 +354 1 +360 1 +362 1 +373 1 +384 1 +395 1 +396 1 +402 1 +417 1 +42 1 +43 1 +442 1 +444 1 +449 1 +453 1 +459 1 +47 1 +48 1 +481 1 +482 1 +483 1 +485 1 +489 1 +494 1 +502 1 +541 1 +562 1 +574 1 +593 1 +60 1 +621 1 +631 1 +65 1 +653 1 +663 1 +664 1 +681 1 +682 1 +683 1 +713 1 +72 1 +74 1 +742 1 +76 1 +763 1 +771 1 +781 1 +79 1 +791 1 +811 1 +812 1 +82 1 +834 1 +852 1 +86 1 +861 1 +87 1 +872 1 +894 1 +903 1 +912 1 +914 1 +932 1 +942 1 +944 1 +96 1 +963 1 +record_5 1 +0 1 +021 1 +064 1 +08 1 +081 1 +082 1 +084 1 +094 1 +118 1 +119 1 +12 1 +122 1 +125 1 +153 1 +156 1 +163 1 +164 1 +167 1 +168 1 +175 1 +176 1 +183 1 +184 1 +190 1 +193 1 +205 1 +209 1 +221 1 +234 1 +243 1 +247 1 +249 1 +251 1 +271 1 +272 1 +281 1 +283 1 +289 1 +306 1 +309 1 +315 1 +317 1 +318 1 +322 1 +331 1 +34 1 +341 1 +345 1 +365 1 +366 1 +372 1 +374 1 +375 1 +379 1 +38 1 +382 1 +391 1 +399 1 +400 1 +401 1 +406 1 +41 1 +413 1 +431 1 +44 1 +443 1 +446 1 +452 1 +463 1 +466 1 +467 1 +469 1 +470 1 +472 1 +473 1 +491 1 +498 1 +5 1 +51 1 +54 1 +543 1 +554 1 +561 1 +572 1 +584 1 +594 1 +612 1 +613 1 +64 1 +662 1 +68 1 +692 1 +70 1 +702 1 +704 1 +714 1 +724 1 +773 1 +774 1 +782 1 +802 1 +81 1 +821 1 +832 1 +84 1 +842 1 +851 1 +873 1 +892 1 +933 1 +941 1 +971 1 +973 1 +98 1 +981 1 +984 1 +991 1 +record_1 1 +record_3 1 diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out index a02a57c00b..bc980b6bc5 100644 --- a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out @@ -463,3 +463,61 @@ Stage-0 TableScan [TS_3] (rows=500/500 width=178) default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] +PREHOOK: query: SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: cat +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: cat +POSTHOOK: Input: default@src +#### A masked pattern was here #### +PREHOOK: query: explain analyze +SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze +SELECT +TRANSFORM(a.key, a.value) USING 'cat' AS (tkey, tvalue) +FROM src a join src b +on a.key = b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 + File Output Operator [FS_9] + Transform Operator [SCR_8] (rows=809/1028 width=178) + command:cat + Merge Join Operator [MERGEJOIN_14] (rows=809/1028 width=178) + Conds:RS_3.key=RS_5.key(Inner),Output:["_col0","_col1"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_3] + PartitionCols:key + Filter Operator [FIL_12] (rows=500/500 width=178) + predicate:key is not null + TableScan [TS_0] (rows=500/500 width=178) + default@src,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 3 [SIMPLE_EDGE] + SHUFFLE [RS_5] + PartitionCols:key + Filter Operator [FIL_13] (rows=500/500 width=87) + predicate:key is not null + TableScan [TS_1] (rows=500/500 width=87) + default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] + diff --git a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out b/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out deleted file mode 100644 index 3b3983f896..0000000000 --- a/ql/src/test/results/clientpositive/tez/unionDistinct_2.q.out +++ /dev/null @@ -1,553 +0,0 @@ -PREHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@u1 -POSTHOOK: query: CREATE TABLE u1 as select key, value from src order by key limit 5 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@u1 -POSTHOOK: Lineage: u1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: u1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@u2 -POSTHOOK: query: CREATE TABLE u2 as select key, value from src order by key limit 3 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@u2 -POSTHOOK: Lineage: u2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: u2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 -PREHOOK: type: CREATETABLE_AS_SELECT -PREHOOK: Input: default@src -PREHOOK: Output: database:default -PREHOOK: Output: default@u3 -POSTHOOK: query: CREATE TABLE u3 as select key, value from src order by key desc limit 5 -POSTHOOK: type: CREATETABLE_AS_SELECT -POSTHOOK: Input: default@src -POSTHOOK: Output: database:default -POSTHOOK: Output: default@u3 -POSTHOOK: Lineage: u3.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: u3.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: select * from u1 -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -#### A masked pattern was here #### -POSTHOOK: query: select * from u1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: select * from u2 -PREHOOK: type: QUERY -PREHOOK: Input: default@u2 -#### A masked pattern was here #### -POSTHOOK: query: select * from u2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u2 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -PREHOOK: query: select * from u3 -PREHOOK: type: QUERY -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select * from u3 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union all -select key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union all -select key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union distinct -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union distinct -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -97 val_97 -98 val_98 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union -select key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union all -select key, value from u2 -union -select key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select key, value from -( -select key, value from u1 -union -select key, value from u2 -union -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: drop view if exists v -PREHOOK: type: DROPVIEW -POSTHOOK: query: drop view if exists v -POSTHOOK: type: DROPVIEW -PREHOOK: query: create view v as select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: create view v as select distinct * from -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.key EXPRESSION [(u1)u1.FieldSchema(name:key, type:string, comment:null), (u2)u2.FieldSchema(name:key, type:string, comment:null), (u3)u3.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: v.value EXPRESSION [(u1)u1.FieldSchema(name:value, type:string, comment:null), (u2)u2.FieldSchema(name:value, type:string, comment:null), (u3)u3.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended v -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v -POSTHOOK: query: describe extended v -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v -key string -value string - -#### A masked pattern was here #### -( -select distinct * from u1 -union -select key, value from u2 -union all -select key as key, value from u3 -) tab, viewExpandedText:select distinct `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select `u2`.`key`, `u2`.`value` from `default`.`u2` -union all -select `u3`.`key` as `key`, `u3`.`value` from `default`.`u3` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) -PREHOOK: query: select * from v -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: select * from v -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -POSTHOOK: Input: default@v -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -PREHOOK: query: drop view if exists v -PREHOOK: type: DROPVIEW -PREHOOK: Input: default@v -PREHOOK: Output: default@v -POSTHOOK: query: drop view if exists v -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: default@v -POSTHOOK: Output: default@v -PREHOOK: query: create view v as select tab.* from -( -select distinct * from u1 -union -select distinct * from u2 -) tab -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: create view v as select tab.* from -( -select distinct * from u1 -union -select distinct * from u2 -) tab -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.key EXPRESSION [(u1)u1.FieldSchema(name:key, type:string, comment:null), (u2)u2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: v.value EXPRESSION [(u1)u1.FieldSchema(name:value, type:string, comment:null), (u2)u2.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended v -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v -POSTHOOK: query: describe extended v -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v -key string -value string - -#### A masked pattern was here #### -( -select distinct * from u1 -union -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) -PREHOOK: query: select * from v -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: select * from v -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@v -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: drop view if exists v -PREHOOK: type: DROPVIEW -PREHOOK: Input: default@v -PREHOOK: Output: default@v -POSTHOOK: query: drop view if exists v -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: default@v -POSTHOOK: Output: default@v -PREHOOK: query: create view v as select * from -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: create view v as select * from -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.key EXPRESSION [(u1)u1.FieldSchema(name:key, type:string, comment:null), (u2)u2.FieldSchema(name:key, type:string, comment:null), ] -POSTHOOK: Lineage: v.value EXPRESSION [(u1)u1.FieldSchema(name:value, type:string, comment:null), (u2)u2.FieldSchema(name:value, type:string, comment:null), ] -PREHOOK: query: describe extended v -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@v -POSTHOOK: query: describe extended v -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@v -key string -value string - -#### A masked pattern was here #### -( -select distinct u1.* from u1 -union all -select distinct * from u2 -) tab, viewExpandedText:select `tab`.`key`, `tab`.`value` from -( -select distinct `u1`.`key`, `u1`.`value` from `default`.`u1` -union all -select distinct `u2`.`key`, `u2`.`value` from `default`.`u2` -) `tab`, tableType:VIRTUAL_VIEW, rewriteEnabled:false) -PREHOOK: query: select * from v -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: select * from v -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@v -#### A masked pattern was here #### -0 val_0 -0 val_0 -10 val_10 -100 val_100 -PREHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union -select key as key, value from u3 -) tab -PREHOOK: type: QUERY -PREHOOK: Input: default@u1 -PREHOOK: Input: default@u2 -PREHOOK: Input: default@u3 -#### A masked pattern was here #### -POSTHOOK: query: select distinct * from -( -select key, value from u1 -union all -select key, value from u2 -union -select key as key, value from u3 -) tab -POSTHOOK: type: QUERY -POSTHOOK: Input: default@u1 -POSTHOOK: Input: default@u2 -POSTHOOK: Input: default@u3 -#### A masked pattern was here #### -0 val_0 -10 val_10 -100 val_100 -96 val_96 -97 val_97 -98 val_98 -- 2.13.6 (Apple Git-96)